2019-12-07T19:33:18.401+0000 7fda64acbbc0 0 ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev), process ceph-osd, pid 33936 2019-12-07T19:33:18.404+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) set_cache_shards 1 2019-12-07T19:33:18.404+0000 7fda64acbbc0 -1 auth: error reading file: /var/lib/ceph/osd/ceph-6/keyring: can't open /var/lib/ceph/osd/ceph-6/keyring: (2) No such file or directory 2019-12-07T19:33:18.404+0000 7fda64acbbc0 -1 created new key in keyring /var/lib/ceph/osd/ceph-6/keyring 2019-12-07T19:33:18.404+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) mkfs path /var/lib/ceph/osd/ceph-6 2019-12-07T19:33:18.404+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.404+0000 7fda64acbbc0 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2019-12-07T19:33:18.404+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.404+0000 7fda64acbbc0 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2019-12-07T19:33:18.404+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.404+0000 7fda64acbbc0 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2019-12-07T19:33:18.405+0000 7fda64acbbc0 -1 bluestore(/var/lib/ceph/osd/ceph-6) _read_fsid unparsable uuid 2019-12-07T19:33:18.405+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) mkfs generated fsid 429b1431-873f-4985-95c1-51e7035621d2 2019-12-07T19:33:18.405+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _setup_block_symlink_or_file name block path size 96636764160 create=1 2019-12-07T19:33:18.405+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _setup_block_symlink_or_file resized block file to 90 GiB 2019-12-07T19:33:18.405+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _setup_block_symlink_or_file name block.wal path size 100663296 create=0 2019-12-07T19:33:18.405+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _setup_block_symlink_or_file name block.db path size 0 create=0 2019-12-07T19:33:18.405+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:18.405+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:18.405+0000 7fda64acbbc0 0 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:18.405+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:18.405+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 0 meta) 2019-12-07T19:33:18.405+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:18.405+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db kv_backend = rocksdb 2019-12-07T19:33:18.405+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db do_bluefs = 1 2019-12-07T19:33:18.405+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db initializing bluefs 2019-12-07T19:33:18.405+0000 7fda64acbbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:18.405+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:18.405+0000 7fda64acbbc0 1 bdev(0x560b569d8380 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:18.405+0000 7fda64acbbc0 0 bdev(0x560b569d8380 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:18.406+0000 7fda64acbbc0 1 bdev(0x560b569d8380 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:18.406+0000 7fda64acbbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:18.406+0000 7fda64acbbc0 1 bluefs _add_block_extent bdev 1 0xacccc0000~e6670000 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _add_block_extent done 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log want_seq 0 txn(seq 0 len 0x0 crc 0xffffffff) not dirty, dirty_files empty, no-op 2019-12-07T19:33:18.406+0000 7fda64acbbc0 1 bluefs mkfs osd_uuid 429b1431-873f-4985-95c1-51e7035621d2 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs _init_alloc 2019-12-07T19:33:18.406+0000 7fda64acbbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_add_free done 2019-12-07T19:33:18.406+0000 7fda64acbbc0 1 bluefs mkfs uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _allocate len 0x400000 from 0 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _allocate len 0x400000 from 1 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate 0x400000/10000,400000,0 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate extent: 0xacccc0000~400000/10000,400000,0 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs mkfs op_alloc_add 1 0xacccc0000~e6670000 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55eff900 pos 0x0 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x0 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55eff900 pos now 0x1000 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 done in 0.000016 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 1 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _write_super super block length(encoded): 102 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _write_super superblock 1 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _write_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs _write_super v 1 crc 0x87c9e83d offset 0x1000 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55eff900 type 0 2019-12-07T19:33:18.406+0000 7fda64acbbc0 20 bluefs _stop_alloc 2019-12-07T19:33:18.406+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8c900 shutdown 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs mkfs success 2019-12-07T19:33:18.406+0000 7fda64acbbc0 1 bluefs mount 2019-12-07T19:33:18.406+0000 7fda64acbbc0 10 bluefs _open_super 2019-12-07T19:33:18.407+0000 7fda64acbbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:18.407+0000 7fda64acbbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.407+0000 7fda64acbbc0 20 bluefs _init_alloc 2019-12-07T19:33:18.407+0000 7fda64acbbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:18.407+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:18.407+0000 7fda64acbbc0 10 bluefs _replay 2019-12-07T19:33:18.407+0000 7fda64acbbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.407+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d290 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.407+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:18.408+0000 7fda64acbbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:18.408+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:18.408+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:18.408+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_add_free done 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d290 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.408+0000 7fda64acbbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:18.408+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs _replay 0x1000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs _replay log file size was 0x1000 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs _replay done 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free done 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs mount log write pos set to 0x1000 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs mkdir db 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs mkdir db.slow 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family L: 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family M: 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family P: 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:18.408+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options using custom Env 0x560b569c5b60 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:18.408+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Write Ahead Log file in db.wal: 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.create_if_missing: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.env: 0x560b569c5b60 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.info_log: 0x560b569c4f40 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_manager: 0x560b569a3d10 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs mkdir db 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs mkdir db 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs lock_file dir db (0x560b569c8a20) file LOCK not found, creating 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) with 0x560b55d582c0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs stat dir db (0x560b569c8a20) file CURRENT not found 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: [db/db_impl_open.cc:228] Creating manifest 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_write db/MANIFEST-000001 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs open_for_write mapping db/MANIFEST-000001 vsel_hint 0x1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55eff900 on file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 13 < min_flush_size 524288 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 13 < min_flush_size 524288 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55eff900 file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 0x0~d to file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55eff900 pos 0x0 0x0~d to file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _allocate len 0xd from 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate extent: 0xacd0c0000~10000/10000,10000,0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 2 (was clean) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd0c0000~10000 x_off 0x0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0xd and padding block with 0xff3 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55eff900 pos now 0xd 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 done in 0.000000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (2) on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]), flushing log 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 2 len 0xa5 crc 0x4276fb9a) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effb80 0x1000~1000 to file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effb80 pos 0x1000 0x1000~1000 to file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effb80 pos now 0x2000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 done in 0.000018 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 2 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _truncate 0xd file file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55eff900 type 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_write db/000001.dbtmp 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs open_for_write mapping db/000001.dbtmp vsel_hint 0x1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55eff900 on file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55eff900 file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 0x0~10 to file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55eff900 pos 0x0 0x0~10 to file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 3 (was clean) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55eff900 pos now 0x10 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 done in 0.000000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (3) on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 3 len 0x57 crc 0xb2966c8c) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effb80 0x2000~1000 to file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effb80 pos 0x2000 0x2000~1000 to file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effb80 pos now 0x3000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 done in 0.000012 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 3 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55eff900 type 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs rename db/000001.dbtmp -> db/CURRENT 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs rename db/CURRENT file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs sync_metadata 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effb80 0x3000~1000 to file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effb80 pos 0x3000 0x3000~1000 to file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effb80 pos now 0x4000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 done in 0.000017 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 4 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs sync_metadata done in 0.000078 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0x4000 expected 3000 ratio 1.33333 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs stat dir db (0x560b569c8a20) file IDENTITY not found 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_write db/000000.dbtmp 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs open_for_write mapping db/000000.dbtmp vsel_hint 0x1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55eff900 on file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55eff900 file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 0x0~25 to file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55eff900 pos 0x0 0x0~25 to file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _allocate len 0x25 from 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate extent: 0xacd0e0000~10000/10000,10000,0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 5 (was clean) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd0e0000~10000 x_off 0x0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x25 and padding block with 0xfdb 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55eff900 pos now 0x25 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 done in 0.000000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (5) on file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]), flushing log 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 5 len 0x57 crc 0xabef9f0b) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effb80 0x4000~1000 to file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effb80 pos 0x4000 0x4000~1000 to file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effb80 pos now 0x5000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 done in 0.000014 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 5 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55eff900 type 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs rename db/000000.dbtmp -> db/IDENTITY 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs rename db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d290 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:18.409+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560b55d58268) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x560b55d8cd10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:18.409+0000 7fda64acbbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs mkdir db 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs open_for_write db.wal/000003.log 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs open_for_write mapping db.wal/000003.log vsel_hint 0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55eff900 on file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs sync_metadata 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 6 len 0x69 crc 0xf98f13fc) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effb80 0x5000~1000 to file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effb80 pos 0x5000 0x5000~1000 to file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x5000 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effb80 pos now 0x6000 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 done in 0.000016 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 6 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs sync_metadata done in 0.000107 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0x6000 expected 3000 ratio 2 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs open_for_write db/OPTIONS-000004.dbtmp 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs open_for_write mapping db/OPTIONS-000004.dbtmp vsel_hint 0x1 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55effcc0 on file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 ignoring, length 4948 < min_flush_size 524288 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55effcc0 file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 0x0~1354 to file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effcc0 pos 0x0 0x0~1354 to file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _allocate len 0x1354 from 1 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate extent: 0xacd0f0000~10000/10000,10000,0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 7 (was clean) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x354 and padding block with 0xcac 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effcc0 pos now 0x1354 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 done in 0.000000 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (7) on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]), flushing log 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 7 len 0x60 crc 0xe745209) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effb80 0x6000~1000 to file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effb80 pos 0x6000 0x6000~1000 to file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x6000 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effb80 pos now 0x7000 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 2019-12-07T19:33:18.410+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 done in 0.000019 2019-12-07T19:33:18.410+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 7 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55effcc0 type 0 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs open_for_read db/OPTIONS-000004.dbtmp (sequential) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d340 on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d340 0x0~2000 from file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1354 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0f0000~10000 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _read left 0x2000 len 0x1354 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _read got 4948 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs rename db/OPTIONS-000004.dbtmp -> db/OPTIONS-000005 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs rename db/OPTIONS-000005 file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.411+0000 7fda64acbbc0 4 rocksdb: DB pointer 0x560b5698e800 2019-12-07T19:33:18.411+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _open_fm initializing freespace 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 freelist create size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 freelist allocate 0x0~2000 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _open_fm reserved 0xbb3330000 for bluefs 2019-12-07T19:33:18.411+0000 7fda64acbbc0 1 freelist init 2019-12-07T19:33:18.411+0000 7fda53374700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:18.411+0000 7fda53374700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_ondisk_format_super ondisk_format 3 min_compat_ondisk_format 3 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 311 < min_flush_size 524288 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 311 < min_flush_size 524288 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55eff900 file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 0x0~137 to file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55eff900 pos 0x0 0x0~137 to file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _allocate len 0x137 from 0 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _allocate len 0x137 from 1 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 allocate extent: 0xacd100000~10000/10000,10000,0 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 8 (was clean) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x0 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x137 and padding block with 0xec9 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55eff900 pos now 0x137 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 done in 0.000000 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (8) on file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]), flushing log 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 8 len 0x64 crc 0x17b8e892) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effb80 0x7000~1000 to file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effb80 pos 0x7000 0x7000~1000 to file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x7000 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effb80 pos now 0x8000 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 2019-12-07T19:33:18.411+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 done in 0.000015 2019-12-07T19:33:18.411+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.412+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 8 2019-12-07T19:33:18.412+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.412+0000 7fda64acbbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:18.412+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0x8000 expected 3000 ratio 2.66667 2019-12-07T19:33:18.412+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.412+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 0 meta) 2019-12-07T19:33:18.412+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 1 meta) 2019-12-07T19:33:18.412+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.412+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 1 meta) 2019-12-07T19:33:18.412+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 2 meta) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _sync_bluefs_and_fm bluefs_extents at KV is now 0x[acccc0000~e6670000] 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 106 < min_flush_size 524288 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 106 < min_flush_size 524288 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55eff900 file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 0x137~6a to file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55eff900 pos 0x137 0x137~6a to file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x137 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range using partial tail 0x137 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x1a1 and padding block with 0xe5f 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55eff900 pos now 0x1a1 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55eff900 done in 0.000000 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _close_fm 2019-12-07T19:33:18.413+0000 7fda64acbbc0 1 freelist shutdown 2019-12-07T19:33:18.413+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush 0x560b55eff900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _truncate 0x1a1 file file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55eff900 type 1 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs unlock_file 0x560b55d582c0 on file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:18.413+0000 7fda64acbbc0 1 bluefs umount 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs sync_metadata 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 9 len 0x23 crc 0xa8730e98) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effb80 0x8000~1000 to file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effb80 pos 0x8000 0x8000~1000 to file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x8000 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effb80 pos now 0x9000 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effb80 done in 0.000016 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 9 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs sync_metadata done in 0.000088 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:18.413+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55effb80 type 0 2019-12-07T19:33:18.413+0000 7fda64acbbc0 20 bluefs _stop_alloc 2019-12-07T19:33:18.413+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8c900 shutdown 2019-12-07T19:33:18.413+0000 7fda64acbbc0 1 bdev(0x560b569d8380 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:18.668+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck check (regular) 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:18.910+0000 7fda64acbbc0 0 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 2 meta) 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 2 meta) 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 2 meta) 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db kv_backend = rocksdb 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 2 meta) 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db do_bluefs = 1 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db initializing bluefs 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bdev(0x560b569d8e00 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:18.910+0000 7fda64acbbc0 0 bdev(0x560b569d8e00 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bdev(0x560b569d8e00 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bluefs mount 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluefs _open_super 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.910+0000 7fda64acbbc0 20 bluefs _init_alloc 2019-12-07T19:33:18.910+0000 7fda64acbbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluefs _replay 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.910+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.910+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_add_free done 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x4276fb9a) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0xb2966c8c) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0xabef9f0b) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xf98f13fc) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xe745209) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0x17b8e892) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0xa8730e98) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d1e0 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _replay done 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free done 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free 0xacd100000~10000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free done 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free done 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free done 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free done 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8c900 init_rm_free done 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family L: 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family M: 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family P: 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:18.911+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options using custom Env 0x560b569c6180 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d550 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d550 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d550 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d550 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d550 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:18.911+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:18.911+0000 7fda64acbbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:18.911+0000 7fda64acbbc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.911+0000 7fda64acbbc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.env: 0x560b569c6180 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.info_log: 0x560b569c6200 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_manager: 0x560b55dd9500 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d550 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d550 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d550 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d550 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560b55d58428) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x560b55d8ce10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d550 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747198913850, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d6b0 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d6b0 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 20 bluefs _read got 417 2019-12-07T19:33:18.912+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747198913889, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:18.912+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old nid_max 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old blobid_max 0 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta freelist_type bitmap 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta per_pool_omap=1 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:18.912+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_csum csum_type crc32c 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:18.913+0000 7fda64acbbc0 1 freelist init 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 freelist init size 0x1680000000 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:18.913+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc opening allocation metadata 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 freelist enumerate_next start 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:18.913+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_add_free 0x2000~167fffe000 2019-12-07T19:33:18.914+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_add_free done 2019-12-07T19:33:18.914+0000 7fda64acbbc0 10 freelist enumerate_next end 2019-12-07T19:33:18.914+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:18.914+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:18.914+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free done 2019-12-07T19:33:18.914+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_collections 2019-12-07T19:33:18.914+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 1 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:18.914+0000 7fda53374700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263454 2019-12-07T19:33:18.914+0000 7fda53374700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:18.914+0000 7fda53374700 5 bluestore.MempoolThread(0x560b569caa08) _resize_shards cache_size: 2845356990 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:18.917+0000 7fda64acbbc0 1 bluefs fsck 2019-12-07T19:33:18.917+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:18.917+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open walking object keyspace 2019-12-07T19:33:18.918+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking shared_blobs 2019-12-07T19:33:18.919+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking pool_statfs 2019-12-07T19:33:18.919+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking for stray omap data 2019-12-07T19:33:18.919+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking deferred events 2019-12-07T19:33:18.919+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:18.920+0000 7fda64acbbc0 10 freelist enumerate_next start 2019-12-07T19:33:18.920+0000 7fda64acbbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:18.964+0000 7fda53374700 20 bluestore.MempoolThread(0x560b569caa08) _resize_shards cache_size: 2845356990 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:18.982+0000 7fda64acbbc0 10 freelist enumerate_next end 2019-12-07T19:33:18.983+0000 7fda64acbbc0 2 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open 0 objects, 0 of them sharded. 2019-12-07T19:33:18.983+0000 7fda64acbbc0 2 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open 0 extents to 0 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:18.983+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.069178 seconds 2019-12-07T19:33:18.983+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _flush_cache 2019-12-07T19:33:18.983+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:18.983+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:18.983+0000 7fda64acbbc0 1 bluefs umount 2019-12-07T19:33:18.983+0000 7fda64acbbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:18.983+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:18.984+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55eff900 type 0 2019-12-07T19:33:18.984+0000 7fda64acbbc0 20 bluefs _stop_alloc 2019-12-07T19:33:18.984+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8c900 shutdown 2019-12-07T19:33:18.984+0000 7fda64acbbc0 1 bdev(0x560b569d8e00 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:19.165+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8cf00 shutdown 2019-12-07T19:33:19.165+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _close_fm 2019-12-07T19:33:19.165+0000 7fda64acbbc0 1 freelist shutdown 2019-12-07T19:33:19.165+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 2 meta) 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.415+0000 7fda64acbbc0 0 bluestore(/var/lib/ceph/osd/ceph-6) mkfs success 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) set_cache_shards 1 2019-12-07T19:33:19.415+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _mount path /var/lib/ceph/osd/ceph-6 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.415+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck check (regular) 2019-12-07T19:33:19.415+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:19.415+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:19.415+0000 7fda64acbbc0 0 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:19.415+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.415+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db kv_backend = rocksdb 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db do_bluefs = 1 2019-12-07T19:33:19.415+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db initializing bluefs 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:19.416+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:19.416+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:19.416+0000 7fda64acbbc0 0 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:19.416+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:19.416+0000 7fda64acbbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:19.416+0000 7fda64acbbc0 1 bluefs mount 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 bluefs _open_super 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.416+0000 7fda64acbbc0 20 bluefs _init_alloc 2019-12-07T19:33:19.416+0000 7fda64acbbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 bluefs _replay 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.416+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:19.416+0000 7fda64acbbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:19.416+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:19.416+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:19.416+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:19.416+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_add_free done 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x4276fb9a) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0xb2966c8c) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0xabef9f0b) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xf98f13fc) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xe745209) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0x17b8e892) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0xa8730e98) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _replay done 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free done 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free 0xacd100000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free done 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free done 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free done 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free done 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cf00 init_rm_free done 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family L: 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family M: 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family P: 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options using custom Env 0x560b569c4f80 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d970 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d970 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:19.417+0000 7fda64acbbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.env: 0x560b569c4f80 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.info_log: 0x560b569c4e80 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_manager: 0x560b55dd9b60 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d970 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d970 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560b55d58478) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x560b55d8c910 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:19.417+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:19.417+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:19.417+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:19.418+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:19.418+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747199419212, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:19.418+0000 7fda64acbbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d810 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d810 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.418+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:19.418+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:19.418+0000 7fda64acbbc0 20 bluefs _read got 417 2019-12-07T19:33:19.418+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747199419249, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:19.418+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old nid_max 0 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old blobid_max 0 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta freelist_type bitmap 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta per_pool_omap=1 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_csum csum_type crc32c 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 1 freelist init 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 freelist init size 0x1680000000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc opening allocation metadata 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 freelist enumerate_next start 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:19.418+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_add_free 0x2000~167fffe000 2019-12-07T19:33:19.419+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_add_free done 2019-12-07T19:33:19.419+0000 7fda64acbbc0 10 freelist enumerate_next end 2019-12-07T19:33:19.419+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:19.419+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:19.419+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:19.419+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_collections 2019-12-07T19:33:19.419+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 1 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:19.419+0000 7fda5316e700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263439 2019-12-07T19:33:19.419+0000 7fda5316e700 5 bluestore.MempoolThread(0x560b569caa08) _resize_shards cache_size: 2845357399 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:19.421+0000 7fda64acbbc0 1 bluefs fsck 2019-12-07T19:33:19.421+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:19.421+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open walking object keyspace 2019-12-07T19:33:19.422+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking shared_blobs 2019-12-07T19:33:19.422+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking pool_statfs 2019-12-07T19:33:19.422+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking for stray omap data 2019-12-07T19:33:19.422+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking deferred events 2019-12-07T19:33:19.422+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:19.424+0000 7fda64acbbc0 10 freelist enumerate_next start 2019-12-07T19:33:19.424+0000 7fda64acbbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:19.469+0000 7fda5316e700 20 bluestore.MempoolThread(0x560b569caa08) _resize_shards cache_size: 2845357399 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:19.486+0000 7fda64acbbc0 10 freelist enumerate_next end 2019-12-07T19:33:19.487+0000 7fda64acbbc0 2 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open 0 objects, 0 of them sharded. 2019-12-07T19:33:19.487+0000 7fda64acbbc0 2 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open 0 extents to 0 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:19.487+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.068371 seconds 2019-12-07T19:33:19.487+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _flush_cache 2019-12-07T19:33:19.487+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:19.487+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:19.487+0000 7fda64acbbc0 1 bluefs umount 2019-12-07T19:33:19.487+0000 7fda64acbbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:19.487+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:19.487+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55effe00 type 0 2019-12-07T19:33:19.487+0000 7fda64acbbc0 20 bluefs _stop_alloc 2019-12-07T19:33:19.487+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8cf00 shutdown 2019-12-07T19:33:19.487+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:19.671+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8ca00 shutdown 2019-12-07T19:33:19.671+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _close_fm 2019-12-07T19:33:19.671+0000 7fda64acbbc0 1 freelist shutdown 2019-12-07T19:33:19.671+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:19.921+0000 7fda64acbbc0 0 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db kv_backend = rocksdb 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db do_bluefs = 1 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db initializing bluefs 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:19.921+0000 7fda64acbbc0 0 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bluefs mount 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluefs _open_super 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.921+0000 7fda64acbbc0 20 bluefs _init_alloc 2019-12-07T19:33:19.921+0000 7fda64acbbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:19.921+0000 7fda64acbbc0 10 bluefs _replay 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_add_free done 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x4276fb9a) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0xb2966c8c) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0xabef9f0b) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xf98f13fc) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xe745209) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0x17b8e892) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0xa8730e98) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _replay done 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd100000~10000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family L: 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family M: 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family P: 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:19.922+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options using custom Env 0x560b569c6f20 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d760 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d760 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:19.922+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:19.922+0000 7fda64acbbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:19.922+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.env: 0x560b569c6f20 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.info_log: 0x560b569c55e0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_manager: 0x560b55dd9b60 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d760 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d760 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560b55d58420) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x560b55d8cf10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d760 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747199924719, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d810 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d810 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 20 bluefs _read got 417 2019-12-07T19:33:19.923+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747199924763, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:19.923+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old nid_max 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old blobid_max 0 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta freelist_type bitmap 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta per_pool_omap=1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_csum csum_type crc32c 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 1 freelist init 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 freelist init size 0x1680000000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cc00 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc opening allocation metadata 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 freelist enumerate_next start 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:19.923+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cc00 init_add_free 0x2000~167fffe000 2019-12-07T19:33:19.924+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cc00 init_add_free done 2019-12-07T19:33:19.924+0000 7fda64acbbc0 10 freelist enumerate_next end 2019-12-07T19:33:19.924+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:19.924+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cc00 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:19.924+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cc00 init_rm_free done 2019-12-07T19:33:19.924+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:19.924+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:19.924+0000 7fda64acbbc0 1 bluefs umount 2019-12-07T19:33:19.924+0000 7fda64acbbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:19.924+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:19.924+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55effcc0 type 0 2019-12-07T19:33:19.924+0000 7fda64acbbc0 20 bluefs _stop_alloc 2019-12-07T19:33:19.924+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8ca00 shutdown 2019-12-07T19:33:19.924+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db kv_backend = rocksdb 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db do_bluefs = 1 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db initializing bluefs 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:20.175+0000 7fda64acbbc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:20.175+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:20.175+0000 7fda64acbbc0 0 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:20.175+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:20.175+0000 7fda64acbbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:20.175+0000 7fda64acbbc0 1 bluefs mount 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _open_super 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _init_alloc 2019-12-07T19:33:20.175+0000 7fda64acbbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _replay 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_add_free done 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x4276fb9a) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.175+0000 7fda64acbbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0xb2966c8c) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:20.175+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0xabef9f0b) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xf98f13fc) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xe745209) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0x17b8e892) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0xa8730e98) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 4096 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _replay done 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd100000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 init_rm_free done 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family L: 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family M: 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family P: 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options using custom Env 0x560b569c6020 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d8c0 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d8c0 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:20.176+0000 7fda64acbbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.env: 0x560b569c6020 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.info_log: 0x560b569c6e80 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_manager: 0x560b55dd9b30 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs mkdir db 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs mkdir db 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) with 0x560b55d58490 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db/CURRENT file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d8c0 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d8c0 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 16 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d8c0 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560b55d58478) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x560b55d8cf10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d8c0 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 13 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs mkdir db 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747200177863, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:20.176+0000 7fda64acbbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d810 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d810 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _read got 417 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_write db/000004.sst 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs open_for_write mapping db/000004.sst vsel_hint 0x1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55effa40 on file(ino 8 size 0x0 mtime 2019-12-07T19:33:20.177905+0000 allocated 0 extents []) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 1383 < min_flush_size 524288 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55effa40 file(ino 8 size 0x0 mtime 2019-12-07T19:33:20.177905+0000 allocated 0 extents []) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 0x0~567 to file(ino 8 size 0x0 mtime 2019-12-07T19:33:20.177905+0000 allocated 0 extents []) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effa40 pos 0x0 0x0~567 to file(ino 8 size 0x0 mtime 2019-12-07T19:33:20.177905+0000 allocated 0 extents []) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _allocate len 0x567 from 1 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 allocate extent: 0xacd110000~10000/10000,10000,0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 10 (was clean) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd110000~10000 x_off 0x0 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x567 and padding block with 0xa99 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effa40 pos now 0x567 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effa40 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effa40 done in 0.000000 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (10) on file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]), flushing log 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.176+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.176+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 10 len 0x56 crc 0x7a89f707) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 0x9000~1000 to file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effcc0 pos 0x9000 0x9000~1000 to file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x9000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effcc0 pos now 0xa000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 done in 0.000024 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 10 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55effa40 type 2 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d810 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d810 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read got 1383 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _read_random h 0x560b55d5d810 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random got 53 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _read_random h 0x560b55d5d810 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random got 85 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _read_random h 0x560b55d5d810 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random got 696 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _read_random h 0x560b55d5d810 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random got 28 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _read_random h 0x560b55d5d810 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _read_random got 74 2019-12-07T19:33:20.177+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747200178437, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1383, "table_properties": {"data_size": 447, "index_size": 28, "filter_size": 69, "raw_key_size": 314, "raw_average_key_size": 22, "raw_value_size": 127, "raw_average_value_size": 9, "num_data_blocks": 1, "num_entries": 14, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2019-12-07T19:33:20.177+0000 7fda64acbbc0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 5 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_write db/MANIFEST-000005 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs open_for_write mapping db/MANIFEST-000005 vsel_hint 0x1 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55effa40 on file(ino 9 size 0x0 mtime 2019-12-07T19:33:20.178475+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 35 < min_flush_size 524288 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 44 < min_flush_size 524288 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55effa40 file(ino 9 size 0x0 mtime 2019-12-07T19:33:20.178475+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 0x0~66 to file(ino 9 size 0x0 mtime 2019-12-07T19:33:20.178475+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effa40 pos 0x0 0x0~66 to file(ino 9 size 0x0 mtime 2019-12-07T19:33:20.178475+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _allocate len 0x66 from 1 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 allocate extent: 0xacd120000~10000/10000,10000,0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 11 (was clean) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x66 and padding block with 0xf9a 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effa40 pos now 0x66 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effa40 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effa40 done in 0.000000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (11) on file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]), flushing log 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 11 len 0x5a crc 0xfda2d21a) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 0xa000~1000 to file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effcc0 pos 0xa000 0xa000~1000 to file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xa000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effcc0 pos now 0xb000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 done in 0.000017 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 11 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_write db/000005.dbtmp 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs open_for_write mapping db/000005.dbtmp vsel_hint 0x1 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55dc4000 on file(ino 10 size 0x0 mtime 2019-12-07T19:33:20.178749+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55dc4000 file(ino 10 size 0x0 mtime 2019-12-07T19:33:20.178749+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4000 0x0~10 to file(ino 10 size 0x0 mtime 2019-12-07T19:33:20.178749+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55dc4000 pos 0x0 0x0~10 to file(ino 10 size 0x0 mtime 2019-12-07T19:33:20.178749+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 allocate extent: 0xacd130000~10000/10000,10000,0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 12 (was clean) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55dc4000 pos now 0x10 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55dc4000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55dc4000 done in 0.000000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (12) on file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]), flushing log 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 12 len 0x57 crc 0xfa1bec5) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 0xb000~1000 to file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effcc0 pos 0xb000 0xb000~1000 to file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xb000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effcc0 pos now 0xc000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 done in 0.000016 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 12 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55dc4000 type 0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs rename db/000005.dbtmp -> db/CURRENT 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs rename dir db (0x560b569c8ae0) file CURRENT already exists, unlinking 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _drop_link had refs 1 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _drop_link destroying file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs rename db/CURRENT file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747200178965, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_write db.wal/000006.log 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs open_for_write mapping db.wal/000006.log vsel_hint 0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55dc4000 on file(ino 11 size 0x0 mtime 2019-12-07T19:33:20.178974+0000 allocated 0 extents []) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs unlink db.wal/000003.log 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _drop_link had refs 1 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _drop_link destroying file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs unlink db/MANIFEST-000001 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _drop_link had refs 1 on file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _drop_link destroying file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs sync_metadata 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 13 len 0xc8 crc 0x35906d76) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 0xc000~1000 to file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effcc0 pos 0xc000 0xc000~1000 to file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xc000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effcc0 pos now 0xd000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 done in 0.000012 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 13 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 release 0xacd0c0000~20000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 release 0xacd100000~10000 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 release done 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs sync_metadata done in 0.000091 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0xd000 expected 3000 ratio 4.33333 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_write db/OPTIONS-000007.dbtmp 2019-12-07T19:33:20.177+0000 7fda64acbbc0 20 bluefs open_for_write mapping db/OPTIONS-000007.dbtmp vsel_hint 0x1 2019-12-07T19:33:20.177+0000 7fda64acbbc0 10 bluefs open_for_write h 0x560b55dc4140 on file(ino 12 size 0x0 mtime 2019-12-07T19:33:20.179115+0000 allocated 0 extents []) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4140 ignoring, length 4949 < min_flush_size 524288 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _fsync 0x560b55dc4140 file(ino 12 size 0x0 mtime 2019-12-07T19:33:20.179115+0000 allocated 0 extents []) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4140 0x0~1355 to file(ino 12 size 0x0 mtime 2019-12-07T19:33:20.179115+0000 allocated 0 extents []) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55dc4140 pos 0x0 0x0~1355 to file(ino 12 size 0x0 mtime 2019-12-07T19:33:20.179115+0000 allocated 0 extents []) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _allocate len 0x1355 from 1 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8ca00 allocate extent: 0xacd0c0000~10000/10000,10000,0 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_range dirty_seq = 14 (was clean) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacd0c0000~10000 x_off 0x0 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_range caching tail of 0x355 and padding block with 0xcab 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55dc4140 pos now 0x1355 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55dc4140 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55dc4140 done in 0.000000 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _fsync file metadata was dirty (14) on file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]), flushing log 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 14 len 0x60 crc 0xbc72df2d) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 0xd000~1000 to file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effcc0 pos 0xd000 0xd000~1000 to file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xd000 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effcc0 pos now 0xe000 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 done in 0.000013 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 14 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55dc4140 type 0 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs open_for_read db/OPTIONS-000007.dbtmp (sequential) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs open_for_read h 0x560b55d5d970 on file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _read h 0x560b55d5d970 0x0~2000 from file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1355 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0c0000~10000 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _read left 0x2000 len 0x1355 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _read got 4949 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs rename db/OPTIONS-000007.dbtmp -> db/OPTIONS-000008 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs rename db/OPTIONS-000008 file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 4 rocksdb: DB pointer 0x560b5698c400 2019-12-07T19:33:20.178+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:20.178+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _upgrade_super from 3, latest 3 2019-12-07T19:33:20.178+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _upgrade_super done 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_collections 2019-12-07T19:33:20.178+0000 7fda5396f700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:20.178+0000 7fda5396f700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 1.35 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Sum 1/0 1.35 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 1.35 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Sum 1/0 1.35 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluefs _read_random h 0x560b55d5d810 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluefs _read_random got 447 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_start 2019-12-07T19:33:20.178+0000 7fda55172700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread start 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_replay start 2019-12-07T19:33:20.178+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:20.178+0000 7fda5a17c700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread start 2019-12-07T19:33:20.178+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach meta fresh osr 0x560b55dc4140 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_replay draining osr 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_register_zombie 0x560b55dc4140 meta 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all osr_set 0x560b55dc4140 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:20.178+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all drain 0x560b55dc4140 2019-12-07T19:33:20.178+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:20.178+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:20.178+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:20.178+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all reaping empty zombie osr 0x560b55dc4140 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all done 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_replay completed 0 events 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore.OnodeSpace(0x560b55d8b220 in 0x560b569d2000) clear 2019-12-07T19:33:20.178+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach meta fresh osr 0x560b55dc4140 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x560b55d8b0e0 meta 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x560b55dc4140 = 0x560b55da8f00 seq 1 2019-12-07T19:33:20.179+0000 7fda64acbbc0 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection meta bits 0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection meta bits 0 = 0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x560b55d8b0e0) get_onode oid #-1:7b3f43c4:::osd_superblock:0# key 0x7f7fffffffffffffff7b3f43c4216f73'd_superblock!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x560b55d8b0e0) r -2 v.len 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 1 Request: 944 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~233 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:20.179+0000 7fda5997b700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263051 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~233 2019-12-07T19:33:20.179+0000 7fda5997b700 5 bluestore.MempoolThread(0x560b569caa08) _resize_shards cache_size: 2845366651 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 1112 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xdcd on front/back, now 0x0~1000 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x560b55da8f00 1 blobs 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cc00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 fbmap_alloc 0x560b55d8cc00 allocate extent: 0x2000~1000/1000,1000,0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2000~1000] 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x560b569ce850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x560b569ce930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x560b569ce850 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x560b569ce930 sbid 0x0)) 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore.blob(0x560b569ce850) get_ref 0x0~233 Blob(0x560b569ce850 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x560b569ce930 sbid 0x0)) 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore.blob(0x560b569ce850) get_ref init 0x1000, 1000 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~233: 0x0~233 Blob(0x560b569ce850 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x560b569ce930 sbid 0x0)) 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore.BufferSpace(0x560b569ce948 in 0x560b55d5f340) _discard 0x0~1000 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x233 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore.extentmap(0x560b55dab008) dirty_range mark inline shard dirty 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 = 0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x560b55da8f00 cost 8848 (2 ios * 4000 + 848 bytes) 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x560b55da8f00 onodes 0x560b55daaf00 shared_blobs 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore.extentmap(0x560b55dab008) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore.extentmap(0x560b55dab008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x560b55da8f00 allocated 0x[2000~1000] released 0x[] 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 freelist allocate 0x2000~1000 2019-12-07T19:33:20.179+0000 7fda64acbbc0 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x560b55da8f00 prepare 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x560b55da8f00 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 3 meta) 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 4 meta) 2019-12-07T19:33:20.179+0000 7fda5db89700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x560b55da8f00 aio_wait 2019-12-07T19:33:20.179+0000 7fda5db89700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x560b55da8f00 2019-12-07T19:33:20.179+0000 7fda5db89700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x560b55da8f00 io_done 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread new_nid_max 1025 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread new_blobid_max 10240 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush 0x560b55dc4000 ignoring, length 265 < min_flush_size 524288 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x560b55daaf00 had 1 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs get_usage bdev 1 free 3860922368 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998813) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399548 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush 0x560b55dc4000 ignoring, length 288 < min_flush_size 524288 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush 0x560b55dc4000 ignoring, length 288 < min_flush_size 524288 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _fsync 0x560b55dc4000 file(ino 11 size 0x0 mtime 2019-12-07T19:33:20.178974+0000 allocated 0 extents []) 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush 0x560b55dc4000 0x0~120 to file(ino 11 size 0x0 mtime 2019-12-07T19:33:20.178974+0000 allocated 0 extents []) 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush_range 0x560b55dc4000 pos 0x0 0x0~120 to file(ino 11 size 0x0 mtime 2019-12-07T19:33:20.178974+0000 allocated 0 extents []) 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _allocate len 0x120 from 0 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _allocate len 0x120 from 1 2019-12-07T19:33:20.179+0000 7fda55172700 10 fbmap_alloc 0x560b55d8ca00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:20.179+0000 7fda55172700 10 fbmap_alloc 0x560b55d8ca00 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range dirty_seq = 15 (was clean) 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range file now file(ino 11 size 0x10000 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range caching tail of 0x120 and padding block with 0xee0 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 4 meta) 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 5 meta) 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range h 0x560b55dc4000 pos now 0x120 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs wait_for_aio 0x560b55dc4000 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs wait_for_aio 0x560b55dc4000 done in 0.000000 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs flush_bdev 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _fsync file metadata was dirty (15) on file(ino 11 size 0x10000 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_and_sync_log op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush_and_sync_log txn(seq 15 len 0x64 crc 0x3767d7e) 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush 0x560b55effcc0 0xe000~1000 to file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _flush_range 0x560b55effcc0 pos 0xe000 0xe000~1000 to file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range file now file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xe000 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_range h 0x560b55effcc0 pos now 0xf000 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs wait_for_aio 0x560b55effcc0 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs wait_for_aio 0x560b55effcc0 done in 0.000015 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs flush_bdev 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_and_sync_log log_seq_stable 15 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluefs _flush_and_sync_log cleaned file file(ino 11 size 0x10000 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluefs _should_compact_log current 0xf000 expected 3000 ratio 5 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread nid_max now 1025 2019-12-07T19:33:20.179+0000 7fda55172700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread blobid_max now 10240 2019-12-07T19:33:20.179+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:20.179+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x560b55da8f00> 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000366773s (0.000007565s flush + 0.000359208s kv commit) 2019-12-07T19:33:20.179+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:20.179+0000 7fda5a17c700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x560b55da8f00 kv_submitted 2019-12-07T19:33:20.179+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:20.179+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x560b55da8f00 2019-12-07T19:33:20.179+0000 7fda5a17c700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x560b55da8f00 finishing 2019-12-07T19:33:20.179+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x560b55da8f00 onodes 0x560b55daaf00 2019-12-07T19:33:20.179+0000 7fda5a17c700 20 bluestore.BufferSpace(0x560b569ce948 in 0x560b55d5f340) _finish_write discard buffer(0x560b55d83cb0 space 0x560b569ce948 0x0~1000 writing nocache) 2019-12-07T19:33:20.179+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x560b55da8f00 done 2019-12-07T19:33:20.179+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:20.179+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x560b55dc4140 q now empty 2019-12-07T19:33:20.179+0000 7fda5a17c700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x560b55da8f00 [] 2019-12-07T19:33:20.179+0000 7fda5a17c700 10 fbmap_alloc 0x560b55d8cc00 release done 2019-12-07T19:33:20.180+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:20.180+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 5 meta) 2019-12-07T19:33:20.180+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 6 meta) 2019-12-07T19:33:20.181+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:20.181+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 6 meta) 2019-12-07T19:33:20.181+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:20.182+0000 7fda64acbbc0 1 bluestore(/var/lib/ceph/osd/ceph-6) umount 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all osr_set 0x560b55dc4140 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all drain 0x560b55dc4140 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all done 2019-12-07T19:33:20.182+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:20.182+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:20.182+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:20.182+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) umount stopping kv thread 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_stop 2019-12-07T19:33:20.182+0000 7fda55172700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:20.182+0000 7fda5a17c700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:20.182+0000 7fda55172700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread finish 2019-12-07T19:33:20.182+0000 7fda5a17c700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread finish 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_stop stopping finishers 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_stop stopped 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _flush_cache 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluestore.sharedblob(0x560b569ce930) put 0x560b569ce930 removing self from set 0x560b55d8b1b0 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluestore.BufferSpace(0x560b569ce948 in 0x560b55d5f340) _clear 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluestore.OnodeSpace(0x560b55d8b220 in 0x560b569d2000) clear 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluestore(/var/lib/ceph/osd/ceph-6) umount closing 2019-12-07T19:33:20.182+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs readdir db 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs readdir db.wal 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4000 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4000 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4000 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush 0x560b55dc4000 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _truncate 0x120 file file(ino 11 size 0x10000 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55dc4000 type 1 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _truncate 0x66 file file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55effa40 type 0 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs unlock_file 0x560b55d58490 on file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:20.182+0000 7fda64acbbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:20.182+0000 7fda64acbbc0 1 bluefs umount 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs sync_metadata 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush_and_sync_log txn(seq 16 len 0x23 crc 0x45e0f6aa) 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush 0x560b55effcc0 0xf000~1000 to file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _flush_range 0x560b55effcc0 pos 0xf000 0xf000~1000 to file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluefs _flush_range file now file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xf000 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluefs _flush_range h 0x560b55effcc0 pos now 0x10000 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs wait_for_aio 0x560b55effcc0 done in 0.000030 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluefs flush_bdev 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluefs _flush_and_sync_log log_seq_stable 16 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs sync_metadata done in 0.000144 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:20.182+0000 7fda64acbbc0 10 bluefs _close_writer 0x560b55effcc0 type 0 2019-12-07T19:33:20.182+0000 7fda64acbbc0 20 bluefs _stop_alloc 2019-12-07T19:33:20.182+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8ca00 shutdown 2019-12-07T19:33:20.182+0000 7fda64acbbc0 1 bdev(0x560b569d9880 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:20.439+0000 7fda64acbbc0 1 fbmap_alloc 0x560b55d8cc00 shutdown 2019-12-07T19:33:20.439+0000 7fda64acbbc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _close_fm 2019-12-07T19:33:20.439+0000 7fda64acbbc0 1 freelist shutdown 2019-12-07T19:33:20.439+0000 7fda64acbbc0 1 bdev(0x560b569d8000 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:20.684+0000 7fda64acbbc0 0 created object store /var/lib/ceph/osd/ceph-6 for osd.6 fsid ad52b421-bda1-4750-8dc4-c9e4d1297c01 2019-12-07T19:33:34.995+0000 7f2b9bef3bc0 0 ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev), process ceph-osd, pid 36937 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 5 object store type is bluestore 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) set_cache_shards 1 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 0 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 1 bdev(0x5557e631a380 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:34.996+0000 7f2b9bef3bc0 0 bdev(0x5557e631a380 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:34.997+0000 7f2b9bef3bc0 1 bdev(0x5557e631a380 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:34.997+0000 7f2b9bef3bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:34.997+0000 7f2b9bef3bc0 1 bdev(0x5557e631a380 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:35.256+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:35.508+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:35.508+0000 7f2b9bef3bc0 1 objectstore numa_node 0 2019-12-07T19:33:35.509+0000 7f2b9bef3bc0 0 starting osd.6 osd_data /var/lib/ceph/osd/ceph-6 /var/lib/ceph/osd/ceph-6/journal 2019-12-07T19:33:35.509+0000 7f2b9bef3bc0 -1 Falling back to public interface 2019-12-07T19:33:35.510+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] _finish_bind bind my_addrs is [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] 2019-12-07T19:33:35.510+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6818/36937,v1:0.0.0.0:6819/36937] _finish_bind bind my_addrs is [v2:0.0.0.0:6818/36937,v1:0.0.0.0:6819/36937] 2019-12-07T19:33:35.511+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6820/36937,v1:0.0.0.0:6821/36937] _finish_bind bind my_addrs is [v2:0.0.0.0:6820/36937,v1:0.0.0.0:6821/36937] 2019-12-07T19:33:35.512+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6822/36937,v1:0.0.0.0:6823/36937] _finish_bind bind my_addrs is [v2:0.0.0.0:6822/36937,v1:0.0.0.0:6823/36937] 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 0 load: jerasure load: lrc load: isa 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 0 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:35.517+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 osd.6:0.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 osd.6:1.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 osd.6:2.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 osd.6:3.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 osd.6:4.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 osd.6:5.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 osd.6:6.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 osd.6:7.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] start start 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 -- start start 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 -- start start 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6820/36937,v1:0.0.0.0:6821/36937] start start 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6822/36937,v1:0.0.0.0:6823/36937] start start 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6818/36937,v1:0.0.0.0:6819/36937] start start 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 -- start start 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 2 osd.6 0 init /var/lib/ceph/osd/ceph-6 (looks like ssd) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 2 osd.6 0 journal /var/lib/ceph/osd/ceph-6/journal 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) set_cache_shards 32 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _mount path /var/lib/ceph/osd/ceph-6 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck check (regular) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 0 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:35.775+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db kv_backend = rocksdb 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db do_bluefs = 1 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db initializing bluefs 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 1 bdev(0x5557e631a380 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 0 bdev(0x5557e631a380 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 1 bdev(0x5557e631a380 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 1 bluefs mount 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluefs _open_super 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 20 bluefs _init_alloc 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluefs _replay 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.776+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_add_free done 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x4276fb9a) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0xb2966c8c) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0xabef9f0b) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xf98f13fc) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xe745209) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0x17b8e892) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0xa8730e98) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x7a89f707) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:20.177905+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0xfda2d21a) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:20.178475+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0xfa1bec5) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:20.178749+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0x35906d76) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:20.178974+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0xbc72df2d) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:20.179115+0000 allocated 0 extents []) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0x3767d7e) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x45e0f6aa) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 bluefs _replay done 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free done 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free 0xacd130000~10000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free done 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free 0xacd120000~10000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free done 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free 0xacd110000~10000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free done 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free done 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free done 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:35.777+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free done 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282900 init_rm_free done 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family L: 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family M: 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family P: 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options using custom Env 0x5557e6ebb280 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6252000 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read got 16 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read got 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6252000 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read got 102 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.env: 0x5557e6ebb280 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.info_log: 0x5557e6ebcb40 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.write_buffer_manager: 0x5557e7045050 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6252000 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read got 16 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read got 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6252000 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5557e624e620) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x5557e6283010 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6252000 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:35.778+0000 7f2b9bef3bc0 20 bluefs _read got 102 2019-12-07T19:33:35.778+0000 7f2b8b70e700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:35.778+0000 7f2b8b70e700 10 bluefs open_for_read h 0x5557e6253810 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.778+0000 7f2b8b70e700 10 bluefs _read h 0x5557e6253810 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read got 1383 2019-12-07T19:33:35.778+0000 7f2b8b70e700 10 bluefs _read_random h 0x5557e6253810 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read_random got 53 2019-12-07T19:33:35.778+0000 7f2b8b70e700 10 bluefs _read_random h 0x5557e6253810 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read_random got 85 2019-12-07T19:33:35.778+0000 7f2b8b70e700 10 bluefs _read_random h 0x5557e6253810 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:35.778+0000 7f2b8b70e700 20 bluefs _read_random got 696 2019-12-07T19:33:35.779+0000 7f2b8b70e700 10 bluefs _read_random h 0x5557e6253810 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.779+0000 7f2b8b70e700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:35.779+0000 7f2b8b70e700 20 bluefs _read_random got 28 2019-12-07T19:33:35.779+0000 7f2b8b70e700 10 bluefs _read_random h 0x5557e6253810 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.779+0000 7f2b8b70e700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:35.779+0000 7f2b8b70e700 20 bluefs _read_random got 74 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747215780446, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e62533f0 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e62533f0 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 20 bluefs _read got 288 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747215780496, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old nid_max 1025 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old blobid_max 10240 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253810 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 20 bluefs _read_random got 447 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta freelist_type bitmap 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta per_pool_omap=1 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_csum csum_type crc32c 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 1 freelist init 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 freelist init size 0x1680000000 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:35.779+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:35.780+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc opening allocation metadata 2019-12-07T19:33:35.780+0000 7f2b9bef3bc0 10 freelist enumerate_next start 2019-12-07T19:33:35.780+0000 7f2b9bef3bc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:35.780+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_add_free 0x3000~167fffd000 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_add_free done 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 10 freelist enumerate_next end 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_collections 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _open_collections opened meta 0x5557e6280f00 cnode(bits 0) 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach meta fresh osr 0x5557e63f5cc0 2019-12-07T19:33:35.781+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 1 Request: 944 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:35.781+0000 7f2b84700700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263808 2019-12-07T19:33:35.781+0000 7f2b84700700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:35.781+0000 7f2b84700700 5 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845347765 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 128 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:35.784+0000 7f2b9bef3bc0 1 bluefs fsck 2019-12-07T19:33:35.784+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x233/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:35.784+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open walking object keyspace 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_check_objects collection meta cnode(bits 0) 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) fsck_check_objects_shallow #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 20 bluestore.blob(0x5557e6f55f10) get_ref 0x0~233 Blob(0x5557e6f55f10 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76cc310 sbid 0x0)) 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 20 bluestore.blob(0x5557e6f55f10) get_ref init 0x1000, 1000 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) fsck_check_objects_shallow 0x0~233: 0x0~233 Blob(0x5557e6f55f10 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x5557e76cc310 sbid 0x0)) 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 20 bluestore.sharedblob(0x5557e76cc310) put 0x5557e76cc310 removing self from set 0x5557e6280fd0 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 20 bluestore.BufferSpace(0x5557e76cc328 in 0x5557e6269340) _clear 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking shared_blobs 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking pool_statfs 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking for stray omap data 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking deferred events 2019-12-07T19:33:35.785+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:35.787+0000 7f2b9bef3bc0 10 freelist enumerate_next start 2019-12-07T19:33:35.787+0000 7f2b9bef3bc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:35.831+0000 7f2b84700700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845347765 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 128 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:35.853+0000 7f2b9bef3bc0 10 freelist enumerate_next end 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 2 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open 1 objects, 0 of them sharded. 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 2 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open 1 extents to 1 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.073021 seconds 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _flush_cache 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 10 bluestore.OnodeSpace(0x5557e6281040 in 0x5557e62c0000) clear 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 1 bluefs umount 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 10 bluefs _close_writer 0x5557e63f5b80 type 0 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 20 bluefs _stop_alloc 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 1 fbmap_alloc 0x5557e6282900 shutdown 2019-12-07T19:33:35.854+0000 7f2b9bef3bc0 1 bdev(0x5557e631a380 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:36.033+0000 7f2b9bef3bc0 1 fbmap_alloc 0x5557e6283100 shutdown 2019-12-07T19:33:36.033+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _close_fm 2019-12-07T19:33:36.033+0000 7f2b9bef3bc0 1 freelist shutdown 2019-12-07T19:33:36.033+0000 7f2b9bef3bc0 1 bdev(0x5557e631a000 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bdev(0x5557e631ae00 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 bdev(0x5557e631ae00 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bdev(0x5557e631ae00 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db kv_backend = rocksdb 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db do_bluefs = 1 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db initializing bluefs 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bdev(0x5557e631aa80 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 bdev(0x5557e631aa80 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bdev(0x5557e631aa80 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bluefs mount 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _open_super 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _init_alloc 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_add_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x4276fb9a) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0xb2966c8c) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0xabef9f0b) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xf98f13fc) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xe745209) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0x17b8e892) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0xa8730e98) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x7a89f707) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:20.177905+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0xfda2d21a) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:20.178475+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0xfa1bec5) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:20.178749+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0x35906d76) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:20.178974+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0xbc72df2d) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:20.179115+0000 allocated 0 extents []) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0x3767d7e) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x45e0f6aa) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253810 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _replay done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd130000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd120000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd110000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family L: 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family M: 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family P: 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options using custom Env 0x5557e6ebd900 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253340 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253340 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 16 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253340 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 0 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253340 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253340 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 20 bluefs _read got 102 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:36.284+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.env: 0x5557e6ebd900 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.info_log: 0x5557e6ebb3a0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.write_buffer_manager: 0x5557e7045800 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253600 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253600 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read got 16 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253600 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read got 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253600 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5557e624e6b0) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x5557e6282910 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253600 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read got 102 2019-12-07T19:33:36.285+0000 7f2b8b508700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:36.285+0000 7f2b8b508700 10 bluefs open_for_read h 0x5557e6253550 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.285+0000 7f2b8b508700 10 bluefs _read h 0x5557e6253550 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read got 1383 2019-12-07T19:33:36.285+0000 7f2b8b508700 10 bluefs _read_random h 0x5557e6253550 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random got 53 2019-12-07T19:33:36.285+0000 7f2b8b508700 10 bluefs _read_random h 0x5557e6253550 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random got 85 2019-12-07T19:33:36.285+0000 7f2b8b508700 10 bluefs _read_random h 0x5557e6253550 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random got 696 2019-12-07T19:33:36.285+0000 7f2b8b508700 10 bluefs _read_random h 0x5557e6253550 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random got 28 2019-12-07T19:33:36.285+0000 7f2b8b508700 10 bluefs _read_random h 0x5557e6253550 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:36.285+0000 7f2b8b508700 20 bluefs _read_random got 74 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216286713, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253760 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253760 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read got 288 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216286744, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old nid_max 1025 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta old blobid_max 10240 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253550 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 20 bluefs _read_random got 447 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta freelist_type bitmap 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_super_meta per_pool_omap=1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_csum csum_type crc32c 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 1 freelist init 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 freelist init size 0x1680000000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282e00 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc opening allocation metadata 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 freelist enumerate_next start 2019-12-07T19:33:36.285+0000 7f2b9bef3bc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282e00 init_add_free 0x3000~167fffd000 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282e00 init_add_free done 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 10 freelist enumerate_next end 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282e00 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6282e00 init_rm_free done 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 1 bluefs umount 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 10 bluefs _close_writer 0x5557e63f5b80 type 0 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 20 bluefs _stop_alloc 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 1 fbmap_alloc 0x5557e6283100 shutdown 2019-12-07T19:33:36.286+0000 7f2b9bef3bc0 1 bdev(0x5557e631aa80 /var/lib/ceph/osd/ceph-6/block) close 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db kv_backend = rocksdb 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db do_bluefs = 1 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_db initializing bluefs 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 1 bdev create path /var/lib/ceph/osd/ceph-6/block type kernel 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 1 bdev(0x5557e631aa80 /var/lib/ceph/osd/ceph-6/block) open path /var/lib/ceph/osd/ceph-6/block 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 0 bdev(0x5557e631aa80 /var/lib/ceph/osd/ceph-6/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-6/block failed: (22) Invalid argument 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 1 bdev(0x5557e631aa80 /var/lib/ceph/osd/ceph-6/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-6/block size 90 GiB 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 1 bluefs mount 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluefs _open_super 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 20 bluefs _init_alloc 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluefs _replay 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:36.540+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_add_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x4276fb9a) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:18.410325+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:18.410351+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0xb2966c8c) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:18.410568+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:18.410581+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0xabef9f0b) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:18.410910+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0xf98f13fc) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:18.411534+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xe745209) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:18.411700+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0x17b8e892) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0xa8730e98) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:18.412617+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x7a89f707) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:20.177905+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0xfda2d21a) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:20.178475+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0xfa1bec5) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:20.178749+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0x35906d76) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:20.178974+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0xbc72df2d) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:20.179115+0000 allocated 0 extents []) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:20.179455+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0x3767d7e) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x45e0f6aa) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 4096 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid b7c9720d-8478-421a-a6d9-fed9288c6d37 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _replay done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd130000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd120000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd110000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 init_rm_free done 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family L: 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family M: 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) column family P: 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options using custom Env 0x5557e6ebb4e0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253550 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 16 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253550 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 102 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.env: 0x5557e6ebb4e0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.info_log: 0x5557e6ebd3c0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.write_buffer_manager: 0x5557e7044e40 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs mkdir db 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs mkdir db 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs mkdir db.wal 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:18.410309+0000 allocated 0 extents []) with 0x5557e624e650 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/CURRENT file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253550 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:18.410923+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253550 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 16 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253550 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5557e624e6b0) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x5557e6282910 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253550 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs _read got 102 2019-12-07T19:33:36.541+0000 7f2b844fa700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:36.541+0000 7f2b844fa700 10 bluefs open_for_read h 0x5557e6253c30 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.541+0000 7f2b844fa700 10 bluefs _read h 0x5557e6253c30 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read got 1383 2019-12-07T19:33:36.541+0000 7f2b844fa700 10 bluefs _read_random h 0x5557e6253c30 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random got 53 2019-12-07T19:33:36.541+0000 7f2b844fa700 10 bluefs _read_random h 0x5557e6253c30 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random got 85 2019-12-07T19:33:36.541+0000 7f2b844fa700 10 bluefs _read_random h 0x5557e6253c30 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random got 696 2019-12-07T19:33:36.541+0000 7f2b844fa700 10 bluefs _read_random h 0x5557e6253c30 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random got 28 2019-12-07T19:33:36.541+0000 7f2b844fa700 10 bluefs _read_random h 0x5557e6253c30 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:36.541+0000 7f2b844fa700 20 bluefs _read_random got 74 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs mkdir db 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:36.541+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216543194, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253b80 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253b80 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read got 288 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_write db/000007.sst 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs open_for_write mapping db/000007.sst vsel_hint 0x1 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_write h 0x5557e76ce140 on file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.543239+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 ignoring, length 1238 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _fsync 0x5557e76ce140 file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.543239+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 0x0~4d6 to file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.543239+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e76ce140 pos 0x0 0x0~4d6 to file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.543239+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _allocate len 0x4d6 from 1 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd100000~10000/10000,10000,0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range dirty_seq = 17 (was clean) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range caching tail of 0x4d6 and padding block with 0xb2a 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e76ce140 pos now 0x4d6 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e76ce140 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e76ce140 done in 0.000000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _fsync file metadata was dirty (17) on file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]), flushing log 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_and_sync_log txn(seq 17 len 0x56 crc 0x425d848b) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e63f5b80 0x10000~1000 to file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e63f5b80 pos 0x10000 0x10000~1000 to file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x10000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x11000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000021 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log log_seq_stable 17 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _close_writer 0x5557e76ce140 type 2 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_read db/000007.sst (random) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253b80 on file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253b80 0x194~342 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) prefetch 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read left 0xe6c len 0x342 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read got 834 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253b80 0x4a1~35 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random left 0xb5f 0x4a1~35 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random got 53 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253b80 0x44c~55 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random left 0xbb4 0x44c~55 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random got 85 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253b80 0x193~2b9 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random left 0xe6d 0x193~2b9 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random got 697 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253b80 0x177~1c from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random left 0xe89 0x177~1c 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random got 28 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253b80 0x12d~4a from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random left 0xed3 0x12d~4a 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _read_random got 74 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216543660, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 1238, "table_properties": {"data_size": 301, "index_size": 28, "filter_size": 69, "raw_key_size": 144, "raw_average_key_size": 24, "raw_value_size": 128, "raw_average_value_size": 21, "num_data_blocks": 1, "num_entries": 6, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 8 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_write db/MANIFEST-000008 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs open_for_write mapping db/MANIFEST-000008 vsel_hint 0x1 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_write h 0x5557e76ce140 on file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.543690+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 ignoring, length 35 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 ignoring, length 87 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _fsync 0x5557e76ce140 file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.543690+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce140 0x0~8f to file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.543690+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e76ce140 pos 0x0 0x0~8f to file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.543690+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _allocate len 0x8f from 1 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd140000~10000/10000,10000,0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range dirty_seq = 18 (was clean) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.543713+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacd140000~10000 x_off 0x0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range caching tail of 0x8f and padding block with 0xf71 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e76ce140 pos now 0x8f 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e76ce140 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e76ce140 done in 0.000000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _fsync file metadata was dirty (18) on file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.543713+0000 allocated 10000 extents [1:0xacd140000~10000]), flushing log 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.543713+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_and_sync_log txn(seq 18 len 0x5b crc 0x2b187d5c) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e63f5b80 0x11000~1000 to file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e63f5b80 pos 0x11000 0x11000~1000 to file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x11000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x12000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000016 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log log_seq_stable 18 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.543713+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_write db/000008.dbtmp 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs open_for_write mapping db/000008.dbtmp vsel_hint 0x1 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_write h 0x5557e76ce280 on file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.543885+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _fsync 0x5557e76ce280 file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.543885+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce280 0x0~10 to file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.543885+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e76ce280 pos 0x0 0x0~10 to file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.543885+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd150000~10000/10000,10000,0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range dirty_seq = 19 (was clean) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.543899+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacd150000~10000 x_off 0x0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x10 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _fsync file metadata was dirty (19) on file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.543899+0000 allocated 10000 extents [1:0xacd150000~10000]), flushing log 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.543899+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_and_sync_log txn(seq 19 len 0x57 crc 0x94912442) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e63f5b80 0x12000~1000 to file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e63f5b80 pos 0x12000 0x12000~1000 to file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x12000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x13000 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000017 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log log_seq_stable 19 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.543899+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _close_writer 0x5557e76ce280 type 0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs rename db/000008.dbtmp -> db/CURRENT 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs rename dir db (0x5557e6f53260) file CURRENT already exists, unlinking 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _drop_link had refs 1 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _drop_link destroying file(ino 10 size 0x10 mtime 2019-12-07T19:33:20.178764+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs rename db/CURRENT file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.543899+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216544130, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_write db.wal/000009.log 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs open_for_write mapping db.wal/000009.log vsel_hint 0 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs open_for_write h 0x5557e76ce280 on file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.544139+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs unlink db.wal/000006.log 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _drop_link had refs 1 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _drop_link destroying file(ino 11 size 0x120 mtime 2019-12-07T19:33:20.180890+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs unlink db/MANIFEST-000005 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _drop_link had refs 1 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs _drop_link destroying file(ino 9 size 0x66 mtime 2019-12-07T19:33:20.178502+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs sync_metadata 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7f2b9bef3bc0 10 bluefs _flush_and_sync_log txn(seq 20 len 0xc8 crc 0x615462e) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e63f5b80 0x13000~1000 to file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e63f5b80 pos 0x13000 0x13000~1000 to file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x13000 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x14000 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000017 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log log_seq_stable 20 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 release 0xacd0d0000~10000 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 release 0xacd120000~20000 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 release done 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs sync_metadata done in 0.000095 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _should_compact_log current 0x14000 expected 3000 ratio 6.66667 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs open_for_write db/OPTIONS-000010.dbtmp 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs open_for_write mapping db/OPTIONS-000010.dbtmp vsel_hint 0x1 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs open_for_write h 0x5557e76ce3c0 on file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.544307+0000 allocated 0 extents []) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce3c0 ignoring, length 4949 < min_flush_size 524288 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _fsync 0x5557e76ce3c0 file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.544307+0000 allocated 0 extents []) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce3c0 0x0~1355 to file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.544307+0000 allocated 0 extents []) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e76ce3c0 pos 0x0 0x0~1355 to file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.544307+0000 allocated 0 extents []) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _allocate len 0x1355 from 1 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range dirty_seq = 21 (was clean) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544638+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range caching tail of 0x355 and padding block with 0xcab 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e76ce3c0 pos now 0x1355 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e76ce3c0 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e76ce3c0 done in 0.000000 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _fsync file metadata was dirty (21) on file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544638+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log op_file_update file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544638+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush_and_sync_log txn(seq 21 len 0x60 crc 0xc818650f) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e63f5b80 0x14000~1000 to file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush_range 0x5557e63f5b80 pos 0x14000 0x14000~1000 to file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range file now file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x14000 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x15000 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000015 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs flush_bdev 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log log_seq_stable 21 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _flush_and_sync_log cleaned file file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544638+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce3c0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _flush 0x5557e76ce3c0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _close_writer 0x5557e76ce3c0 type 0 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs open_for_read db/OPTIONS-000010.dbtmp (sequential) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs open_for_read h 0x5557e6253ad0 on file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544638+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _read h 0x5557e6253ad0 0x0~2000 from file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544638+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1355 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _read left 0x2000 len 0x1355 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _read got 4949 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs rename db/OPTIONS-000010.dbtmp -> db/OPTIONS-000011 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs rename db/OPTIONS-000011 file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544638+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs unlink db/OPTIONS-000005 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _drop_link had refs 1 on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _drop_link destroying file(ino 7 size 0x1354 mtime 2019-12-07T19:33:18.412033+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs readdir db 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs stat db/000007.sst 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs stat db/000007.sst file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 4 rocksdb: DB pointer 0x5557e62bc400 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _upgrade_super from 3, latest 3 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) _upgrade_super done 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _open_collections 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253b80 0x0~12d from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.543341+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _read_random left 0x1000 0x0~12d 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _read_random got 301 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluefs _read_random h 0x5557e6253c30 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:20.178035+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluefs _read_random got 447 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _open_collections opened meta 0x5557e6280960 cnode(bits 0) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach meta fresh osr 0x5557e76ce500 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x233/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_start 2019-12-07T19:33:36.543+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread start 2019-12-07T19:33:36.543+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_replay start 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_replay draining osr 2019-12-07T19:33:36.543+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread start 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_register_zombie 0x5557e76ce500 meta 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all 2019-12-07T19:33:36.543+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all osr_set 0x5557e76ce500 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:36.543+0000 7f2b84cfb700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all drain 0x5557e76ce500 2019-12-07T19:33:36.543+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:36.543+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:36.543+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all reaping empty zombie osr 0x5557e76ce500 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_drain_all done 2019-12-07T19:33:36.543+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:36.543+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_replay completed 0 events 2019-12-07T19:33:36.543+0000 7f2b84cfb700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 2/0 2.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Sum 2/0 2.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 2/0 2.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Sum 2/0 2.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.6 0.00 0.00 1 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) is_journal_rotational 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 2 osd.6 0 journal looks like ssd 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 2 osd.6 0 boot 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 osd.6 0 configured osd_max_object_name[space]_len looks ok 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:7b3f43c4:::osd_superblock:0# 0x0~0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:7b3f43c4:::osd_superblock:0# key 0x7f7fffffffffffffff7b3f43c4216f73'd_superblock!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r 0 v.len 46 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~233 Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore.blob(0x5557e76cc5b0) get_ref init 0x1000, 1000 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~233 size 0x233 (563) 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x5557e76cc700 sbid 0x0)) need 0x0~233 cache has 0x[] 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x5557e76cc700 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~233]} 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x5557e76cc700 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~233]} 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 _add level 1 near 0 on buffer(0x5557e6279ef0 space 0x5557e76cc718 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 = 563 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 osd.6 0 read_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e0 [0,0] lci=[0,0]) 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 osd.6 0 get_map 0 - return initial 0x5557e625f700 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) exists meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c0371625:::snapmapper:0# key 0x7f7fffffffffffffffc03716'%!snapmapper!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 osd.6 0 init creating/touching snapmapper object 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7046f00 seq 1 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c0371625:::snapmapper:0# key 0x7f7fffffffffffffffc03716'%!snapmapper!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1026 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 4205 (1 ios * 4000 + 205 bytes) 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes 0x5557e70ce280 shared_blobs 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 1 Request: 1760 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore.extentmap(0x5557e70ce388) update #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore.extentmap(0x5557e70ce388) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c0371625:::snapmapper:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:36.544+0000 7f2b8ad07700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263462 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:36.544+0000 7f2b8ad07700 5 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 2411 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) exists meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c1a3fc6e:::purged_snaps:0# key 0x7f7fffffffffffffffc1a3fc'n!purged_snaps!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 osd.6 0 init creating/touching purged_snaps object 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7121500 seq 2 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c1a3fc6e:::purged_snaps:0# key 0x7f7fffffffffffffffc1a3fc'n!purged_snaps!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1027 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread new_nid_max 2050 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121500 cost 4207 (1 ios * 4000 + 207 bytes) 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121500 onodes 0x5557e70ce500 shared_blobs 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore.extentmap(0x5557e70ce608) update #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread new_blobid_max 20480 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore.extentmap(0x5557e70ce608) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c1a3fc6e:::purged_snaps:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121500 allocated 0x[] released 0x[] 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 prepare 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121500 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 io_done 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 open_all_classes 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce280 had 1 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 open_all_classes found journal 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399535 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 _get_class adding new class name journal 0x5557e628b348 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 166 < min_flush_size 524288 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 _load_class journal from /usr/lib64/rados-classes/libcls_journal.so 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 166 < min_flush_size 524288 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.544139+0000 allocated 0 extents []) 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x0~a6 to file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.544139+0000 allocated 0 extents []) 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x0 0x0~a6 to file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.544139+0000 allocated 0 extents []) 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _allocate len 0xa6 from 0 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _allocate len 0xa6 from 1 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd120000~10000/10000,10000,0 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 22 (was clean) 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x0 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa6 and padding block with 0xf5a 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0xa6 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (22) on file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]), flushing log 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 22 len 0x86 crc 0x7e412ee6) 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x15000~1000 to file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x15000 0x15000~1000 to file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x15000 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_class journal status 3 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.create flags 3 0x7f2b8a2ef130 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.get_order flags 1 0x7f2b8a2eebf0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.get_splay_width flags 1 0x7f2b8a2eeb90 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.get_pool_id flags 1 0x7f2b8a2ee440 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.get_minimum_set flags 1 0x7f2b8a2eee80 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.set_minimum_set flags 3 0x7f2b8a2ef010 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.get_active_set flags 1 0x7f2b8a2eee10 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.set_active_set flags 3 0x7f2b8a2eeef0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.get_client flags 1 0x7f2b8a2efd90 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.client_register flags 3 0x7f2b8a2f1890 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.client_update_data flags 3 0x7f2b8a2f00e0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.client_update_state flags 3 0x7f2b8a2f0660 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.client_unregister flags 3 0x7f2b8a2f4360 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.client_commit flags 3 0x7f2b8a2f0a70 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.client_list flags 1 0x7f2b8a2f1590 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.get_next_tag_tid flags 1 0x7f2b8a2eeda0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.get_tag flags 1 0x7f2b8a2f4ae0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.tag_create flags 3 0x7f2b8a2f4d80 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.tag_list flags 1 0x7f2b8a2f2570 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.guard_append flags 3 0x7f2b8a2ef490 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method journal.append flags 3 0x7f2b8a2ef5d0 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x16000 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 _load_class journal success 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 open_all_classes found version 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 _get_class adding new class name version 0x5557e628b468 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 _load_class version from /usr/lib64/rados-classes/libcls_version.so 2019-12-07T19:33:36.544+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000019 2019-12-07T19:33:36.544+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_class version status 3 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method version.set flags 3 0x7f2b8a0d3d30 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method version.inc flags 3 0x7f2b8a0d4740 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method version.inc_conds flags 3 0x7f2b8a0d4740 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method version.read flags 1 0x7f2b8a0d3bb0 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 register_cxx_method version.check_conds flags 1 0x7f2b8a0d4590 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 _load_class version success 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 open_all_classes found log 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 _get_class adding new class name log 0x5557e628ada8 2019-12-07T19:33:36.544+0000 7f2b9bef3bc0 10 _load_class log from /usr/lib64/rados-classes/libcls_log.so 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_class log status 3 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method log.add flags 3 0x7f2b89ebcc40 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method log.list flags 1 0x7f2b89ebb4a0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method log.trim flags 3 0x7f2b89eba9e0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method log.info flags 1 0x7f2b89eba4e0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 _load_class log success 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 open_all_classes found rbd 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 _get_class adding new class name rbd 0x5557e628aec8 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 _load_class rbd from /usr/lib64/rados-classes/libcls_rbd.so 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_class rbd status 3 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.create flags 3 0x7f2b89c63c90 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_features flags 1 0x7f2b89c59fc0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_features flags 3 0x7f2b89c5a1a0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_size flags 1 0x7f2b89c6c150 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_size flags 3 0x7f2b89c62200 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_snapcontext flags 1 0x7f2b89c63300 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_object_prefix flags 1 0x7f2b89c5b3b0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_data_pool flags 1 0x7f2b89c5b520 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_snapshot_name flags 1 0x7f2b89c6da20 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_snapshot_timestamp flags 1 0x7f2b89c6dbf0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snapshot_get flags 1 0x7f2b89c6eae0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snapshot_add flags 3 0x7f2b89c6eec0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snapshot_remove flags 3 0x7f2b89c6e430 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snapshot_rename flags 3 0x7f2b89c6dda0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snapshot_trash_add flags 3 0x7f2b89c730d0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_all_features flags 1 0x7f2b89c4cf70 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_parent flags 1 0x7f2b89c6d150 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_parent flags 3 0x7f2b89c61dc0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.remove_parent flags 3 0x7f2b89c6c110 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.parent_get flags 1 0x7f2b89c62dc0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.parent_overlap_get flags 1 0x7f2b89c6d620 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.parent_attach flags 3 0x7f2b89c61ff0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.parent_detach flags 3 0x7f2b89c6c130 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_protection_status flags 3 0x7f2b89c6c700 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_protection_status flags 1 0x7f2b89c6c4c0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_stripe_unit_count flags 1 0x7f2b89c5a540 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_stripe_unit_count flags 3 0x7f2b89c5a7c0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_create_timestamp flags 1 0x7f2b89c5ac40 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_access_timestamp flags 1 0x7f2b89c5ae50 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_modify_timestamp flags 1 0x7f2b89c5b060 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_flags flags 1 0x7f2b89c6c9a0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_flags flags 3 0x7f2b89c6cc60 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.op_features_get flags 1 0x7f2b89c5b270 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.op_features_set flags 3 0x7f2b89c58850 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.metadata_list flags 1 0x7f2b89c65a00 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.metadata_set flags 3 0x7f2b89c733d0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.metadata_remove flags 3 0x7f2b89c524e0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.metadata_get flags 1 0x7f2b89c52640 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snapshot_get_limit flags 1 0x7f2b89c5bc20 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snapshot_set_limit flags 3 0x7f2b89c5cd20 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.child_attach flags 3 0x7f2b89c72310 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.child_detach flags 3 0x7f2b89c71710 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.children_list flags 1 0x7f2b89c72030 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.migration_set flags 3 0x7f2b89c59070 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.migration_set_state flags 3 0x7f2b89c59730 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.migration_get flags 1 0x7f2b89c59920 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.migration_remove flags 3 0x7f2b89c59fa0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_modify_timestamp flags 3 0x7f2b89c5b8f0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_access_timestamp flags 3 0x7f2b89c5b800 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.add_child flags 3 0x7f2b89c74d90 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.remove_child flags 3 0x7f2b89c75170 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_children flags 1 0x7f2b89c755d0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.get_id flags 1 0x7f2b89c4ed20 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.set_id flags 3 0x7f2b89c4ef30 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.dir_get_id flags 1 0x7f2b89c518a0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.dir_get_name flags 1 0x7f2b89c51eb0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.dir_list flags 1 0x7f2b89c5e970 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.dir_add_image flags 3 0x7f2b89c64ec0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.dir_remove_image flags 3 0x7f2b89c51dc0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.dir_rename_image flags 3 0x7f2b89c64d80 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.dir_state_assert flags 1 0x7f2b89c5b9e0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.dir_state_set flags 3 0x7f2b89c5d130 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.object_map_load flags 1 0x7f2b89c66690 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.object_map_save flags 3 0x7f2b89c66ba0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.object_map_resize flags 3 0x7f2b89c67e60 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.object_map_update flags 3 0x7f2b89c67410 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.object_map_snap_add flags 3 0x7f2b89c66760 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.object_map_snap_remove flags 3 0x7f2b89c66d40 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snap_list flags 1 0x7f2b89c5bd60 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snap_add flags 3 0x7f2b89c5c000 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snap_remove flags 3 0x7f2b89c4f110 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.snap_rename flags 3 0x7f2b89c4f650 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_uuid_get flags 1 0x7f2b89c609d0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_uuid_set flags 3 0x7f2b89c63a20 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_mode_get flags 1 0x7f2b89c4dd60 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_mode_set flags 3 0x7f2b89c61070 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_peer_ping flags 3 0x7f2b89c72fc0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_peer_list flags 1 0x7f2b89c5dab0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_peer_add flags 3 0x7f2b89c60dd0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_peer_remove flags 3 0x7f2b89c52e10 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_peer_set_client flags 3 0x7f2b89c55580 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_peer_set_cluster flags 3 0x7f2b89c5dd50 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_peer_set_direction flags 3 0x7f2b89c52c10 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_list flags 1 0x7f2b89c57230 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_get_image_id flags 1 0x7f2b89c53880 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_get flags 1 0x7f2b89c530e0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_set flags 3 0x7f2b89c53780 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_remove flags 3 0x7f2b89c60550 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_status_set flags 11 0x7f2b89c616c0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_status_remove flags 3 0x7f2b89c60610 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_status_get flags 1 0x7f2b89c5e360 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_status_list flags 1 0x7f2b89c69210 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_status_get_summary flags 1 0x7f2b89c69970 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_status_remove_down flags 3 0x7f2b89c57210 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_instance_get flags 1 0x7f2b89c553c0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_instance_list flags 1 0x7f2b89c6a3e0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_instances_list flags 1 0x7f2b89c5e7b0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_instances_add flags 11 0x7f2b89c53c20 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_instances_remove flags 3 0x7f2b89c53dc0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_map_list flags 1 0x7f2b89c56300 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_map_update flags 2 0x7f2b89c53ed0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.mirror_image_map_remove flags 2 0x7f2b89c541b0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_dir_list flags 1 0x7f2b89c5f1d0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_dir_add flags 3 0x7f2b89c65420 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_dir_remove flags 3 0x7f2b89c523f0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_dir_rename flags 3 0x7f2b89c65560 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_image_remove flags 3 0x7f2b89c4d330 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_image_list flags 1 0x7f2b89c57b80 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_image_set flags 3 0x7f2b89c4ddc0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.image_group_add flags 3 0x7f2b89c5c550 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.image_group_remove flags 3 0x7f2b89c5c8b0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.image_group_get flags 1 0x7f2b89c5cb60 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_snap_set flags 3 0x7f2b89c70cf0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_snap_remove flags 3 0x7f2b89c6ff50 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_snap_get_by_id flags 1 0x7f2b89c70080 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.group_snap_list flags 1 0x7f2b89c709e0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.trash_add flags 3 0x7f2b89c656a0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.trash_remove flags 3 0x7f2b89c54340 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.trash_list flags 1 0x7f2b89c6a870 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.trash_get flags 1 0x7f2b89c545b0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.trash_state_set flags 3 0x7f2b89c547d0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.namespace_add flags 3 0x7f2b89c54be0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.namespace_remove flags 3 0x7f2b89c54dc0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.namespace_list flags 1 0x7f2b89c564a0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.copyup flags 3 0x7f2b89c4d130 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.sparse_copyup flags 3 0x7f2b89c740b0 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.assert_snapc_seq flags 3 0x7f2b89c4d010 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 register_cxx_method rbd.sparsify flags 3 0x7f2b89c4e410 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 _load_class rbd success 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 open_all_classes found sdk 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 0 _get_class not permitted to load sdk 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 open_all_classes found rgw 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 _get_class adding new class name rgw 0x5557e628b7c8 2019-12-07T19:33:36.545+0000 7f2b9bef3bc0 10 _load_class rgw from /usr/lib64/rados-classes/libcls_rgw.so 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_class rgw status 3 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_init_index flags 3 0x7f2b89950600 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_set_tag_timeout flags 3 0x7f2b89958340 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_list flags 1 0x7f2b8995eaf0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_check_index flags 1 0x7f2b899605f0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_rebuild_index flags 3 0x7f2b89960470 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_update_stats flags 3 0x7f2b899592c0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_prepare_op flags 3 0x7f2b89961000 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_complete_op flags 3 0x7f2b89961520 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_link_olh flags 3 0x7f2b89963080 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_unlink_instance flags 3 0x7f2b89962570 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_read_olh_log flags 1 0x7f2b8995b0f0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_trim_olh_log flags 3 0x7f2b8995aae0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bucket_clear_olh flags 3 0x7f2b8995a600 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.obj_remove flags 3 0x7f2b8995dc90 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.obj_store_pg_ver flags 2 0x7f2b89953600 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.obj_check_attrs_prefix flags 1 0x7f2b899533c0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.obj_check_mtime flags 1 0x7f2b89953140 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bi_get flags 1 0x7f2b899550b0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bi_put flags 3 0x7f2b89952fc0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bi_list flags 1 0x7f2b8995ba10 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bi_log_list flags 1 0x7f2b89963d60 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bi_log_trim flags 3 0x7f2b89955600 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.dir_suggest_changes flags 3 0x7f2b89959570 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bi_log_resync flags 3 0x7f2b89958ea0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.bi_log_stop flags 3 0x7f2b89958b60 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.user_usage_log_add flags 3 0x7f2b89957330 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.user_usage_log_read flags 1 0x7f2b89956d90 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.user_usage_log_trim flags 3 0x7f2b899571b0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.usage_log_clear flags 2 0x7f2b8994f4d0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.gc_set_entry flags 3 0x7f2b89954c90 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.gc_defer_entry flags 3 0x7f2b89954b70 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.gc_list flags 1 0x7f2b89953a80 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.gc_remove flags 3 0x7f2b89960d20 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.lc_get_entry flags 1 0x7f2b89952d20 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.lc_set_entry flags 3 0x7f2b89953870 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.lc_rm_entry flags 3 0x7f2b899537b0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.lc_get_next_entry flags 1 0x7f2b89952a30 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.lc_put_head flags 3 0x7f2b89952860 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.lc_get_head flags 1 0x7f2b89952610 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.lc_list_entries flags 1 0x7f2b89952070 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.reshard_add flags 3 0x7f2b89951ea0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.reshard_list flags 1 0x7f2b899518e0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.reshard_get flags 1 0x7f2b899516b0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.reshard_remove flags 3 0x7f2b899514b0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.set_bucket_resharding flags 3 0x7f2b899589a0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.clear_bucket_resharding flags 3 0x7f2b89958840 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.guard_bucket_resharding flags 1 0x7f2b899586f0 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 register_cxx_method rgw.get_bucket_resharding flags 1 0x7f2b89958490 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 _load_class rgw success 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 open_all_classes found kvs 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 0 _get_class not permitted to load kvs 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 open_all_classes found refcount 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 _get_class adding new class name refcount 0x5557e628b8e8 2019-12-07T19:33:36.547+0000 7f2b9bef3bc0 10 _load_class refcount from /usr/lib64/rados-classes/libcls_refcount.so 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 22 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 release 0xacd0f0000~10000 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 release done 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs _should_compact_log current 0x16000 expected 3000 ratio 7.33333 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread nid_max now 2050 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread blobid_max now 20480 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.003016396s (0.000000220s flush + 0.003016176s kv commit) 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 0x5557e70ce280 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 125 < min_flush_size 524288 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 125 < min_flush_size 524288 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0xa6~7d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0xa6 0xa6~7d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xa6 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa6 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x123 and padding block with 0xedd 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x123 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:36.547+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000062820s (0.000000095s flush + 0.000062725s kv commit) 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:36.547+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121500> 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121500 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 finishing 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121500 onodes 0x5557e70ce500 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 done 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce500 q now empty 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121500 [] 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:36.547+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish empty zombie osr 0x5557e76ce500 already reaped 2019-12-07T19:33:36.547+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_class refcount status 3 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method refcount.get flags 3 0x7f2b896002e0 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method refcount.put flags 3 0x7f2b89600a10 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method refcount.set flags 3 0x7f2b895ffd90 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method refcount.read flags 1 0x7f2b89601280 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 _load_class refcount success 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 open_all_classes found user 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 _get_class adding new class name user 0x5557e628ba08 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 _load_class user from /usr/lib64/rados-classes/libcls_user.so 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_class user status 3 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method user.set_buckets_info flags 3 0x7f2b8933f800 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method user.complete_stats_sync flags 3 0x7f2b8933bd50 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method user.remove_bucket flags 3 0x7f2b8933b070 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method user.list_buckets flags 1 0x7f2b8933d930 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method user.get_header flags 1 0x7f2b8933ab60 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 register_cxx_method user.reset_user_stats flags 3 0x7f2b8933c360 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 _load_class user success 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 open_all_classes found hello 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 _get_class adding new class name hello 0x5557e628bb28 2019-12-07T19:33:36.548+0000 7f2b9bef3bc0 10 _load_class hello from /usr/lib64/rados-classes/libcls_hello.so 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 0 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/15.0.0-8038-g503c226/rpm/el7/BUILD/ceph-15.0.0-8038-g503c226/src/cls/hello/cls_hello.cc:313: loading cls_hello 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_class hello status 3 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method hello.say_hello flags 1 0x7f2b891224d0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method hello.record_hello flags 10 0x7f2b89123360 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method hello.write_return_data flags 2 0x7f2b89122670 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method hello.write_too_much_return_data flags 2 0x7f2b89122550 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method hello.replay flags 1 0x7f2b891224a0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method hello.turn_it_to_11 flags 11 0x7f2b891229a0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method hello.bad_reader flags 2 0x7f2b89122490 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method hello.bad_writer flags 1 0x7f2b89122480 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class hello success 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 open_all_classes found otp 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _get_class adding new class name otp 0x5557e628bd68 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class otp from /usr/lib64/rados-classes/libcls_otp.so 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_class otp status 3 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method otp.otp_set flags 3 0x7f2b88f054c0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method otp.otp_get flags 1 0x7f2b88f06dd0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method otp.otp_check flags 3 0x7f2b88f03d50 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method otp.otp_get_result flags 1 0x7f2b88f046d0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method otp.otp_remove flags 3 0x7f2b88f08380 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method otp.get_current_time flags 1 0x7f2b88f03740 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class otp success 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 open_all_classes found cas 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _get_class adding new class name cas 0x5557e628be88 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class cas from /usr/lib64/rados-classes/libcls_cas.so 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_class cas status 3 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method cas.chunk_get flags 3 0x7f2b88ad4570 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method cas.chunk_put flags 3 0x7f2b88ad5230 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method cas.chunk_set flags 3 0x7f2b88ad5b60 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method cas.chunk_read flags 1 0x7f2b88ad4b30 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method cas.cas_write_or_get flags 3 0x7f2b88ad4cc0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method cas.has_chunk flags 1 0x7f2b88ad3aa0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class cas success 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 open_all_classes found rgw_gc 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _get_class adding new class name rgw_gc 0x5557e7966148 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class rgw_gc from /usr/lib64/rados-classes/libcls_rgw_gc.so 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_class rgw_gc status 3 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_init flags 3 0x7f2b88892790 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_enqueue flags 3 0x7f2b88892e60 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_list_entries flags 1 0x7f2b88893f70 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_remove_entries flags 3 0x7f2b88899950 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method rgw_gc.rgw_gc_queue_update_entry flags 3 0x7f2b88896ba0 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class rgw_gc success 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 open_all_classes found cephfs 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _get_class adding new class name cephfs 0x5557e7966268 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class cephfs from /usr/lib64/rados-classes/libcls_cephfs.so 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 0 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/15.0.0-8038-g503c226/rpm/el7/BUILD/ceph-15.0.0-8038-g503c226/src/cls/cephfs/cls_cephfs.cc:198: loading cephfs 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_class cephfs status 3 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 register_cxx_method cephfs.accumulate_inode_metadata flags 3 0x7f2b88671900 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class cephfs success 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 open_all_classes found numops 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _get_class adding new class name numops 0x5557e79664a8 2019-12-07T19:33:36.549+0000 7f2b9bef3bc0 10 _load_class numops from /usr/lib64/rados-classes/libcls_numops.so 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_class numops status 3 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method numops.add flags 3 0x7f2b8845bba0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method numops.mul flags 3 0x7f2b8845b170 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 _load_class numops success 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 open_all_classes found lua 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 0 _get_class not permitted to load lua 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 open_all_classes found queue 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 0 _get_class not permitted to load queue 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 open_all_classes found lock 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 _get_class adding new class name lock 0x5557e79665c8 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 _load_class lock from /usr/lib64/rados-classes/libcls_lock.so 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_class lock status 3 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method lock.lock flags 11 0x7f2b882409c0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method lock.unlock flags 11 0x7f2b8823e0a0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method lock.break_lock flags 3 0x7f2b8823d8d0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method lock.get_info flags 1 0x7f2b8823fa50 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method lock.list_locks flags 1 0x7f2b8823bb00 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method lock.assert_locked flags 9 0x7f2b8823ca50 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method lock.set_cookie flags 11 0x7f2b8823e740 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 _load_class lock success 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 open_all_classes found timeindex 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 _get_class adding new class name timeindex 0x5557e7966808 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 _load_class timeindex from /usr/lib64/rados-classes/libcls_timeindex.so 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_class timeindex status 3 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method timeindex.add flags 3 0x7f2b88024080 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method timeindex.list flags 1 0x7f2b88022fe0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 register_cxx_method timeindex.trim flags 3 0x7f2b880226b0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 _load_class timeindex success 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 0 osd.6 0 crush map has features 288232575208783872, adjusting msgr requires for clients 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 0 osd.6 0 crush map has features 288232575208783872 was 8705, adjusting msgr requires for mons 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 0 osd.6 0 crush map has features 288232575208783872, adjusting msgr requires for osds 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 0 osd.6 0 check_osdmap_features enabling on-disk ERASURE CODES compat feature 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 osd.6 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e0 [0,0] lci=[0,0]) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7121800 seq 3 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x233 (563) bytes fadvise_flags 0x0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 _rm buffer(0x5557e6279ef0 space 0x5557e76cc718 0x0~1000 clean) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~233 Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x481) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121800 0 blobs 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~233: 0x0~233 Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x24e 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121800 cost 4803 (1 ios * 4000 + 803 bytes) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121800 onodes 0x5557e62a1b80 shared_blobs 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121800 allocated 0x[] released 0x[] 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 prepare 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121800 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 io_done 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 osd.6 0 create_recoverystate_perf 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4323 < min_flush_size 524288 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 osd.6 0 clear_temp_objects 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 0 osd.6 0 load_pgs 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:46e858f0:::pg_num_history:head# 0x0~0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:46e858f0:::pg_num_history:head# key 0x7f7fffffffffffffff46e858f0217067'_num_history!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:46e858f0:::pg_num_history:head# 0x0~0 = -2 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 osd.6 0 load_pgs pg_num_history pg_num_history(e0 pg_nums {} deleted_pools ) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 osd.6 0 load_pgs ignoring unrecognized meta 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 0 osd.6 0 load_pgs opened 0 pgs 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 2 osd.6 0 superblock: I am osd.6 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 10 osd.6 0 create_logger 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4346 < min_flush_size 524288 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4346 < min_flush_size 524288 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x123~10fa to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x123 0x123~10fa to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x123 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x123 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x21d and padding block with 0xde3 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x121d 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:36.550+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000127450s (0.000000054s flush + 0.000127396s kv commit) 2019-12-07T19:33:36.550+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:36.550+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:36.550+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121800> 2019-12-07T19:33:36.550+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.550+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 kv_submitted 2019-12-07T19:33:36.550+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121800 2019-12-07T19:33:36.550+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7121800 osr 0x5557e76ce500 2019-12-07T19:33:36.550+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e76cf2c0) prepare_write seq 1 0x2000~1000 crc 2a980576 2019-12-07T19:33:36.550+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:36.550+0000 7f2b9bef3bc0 1 Processor -- start 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 -1 osd.6 0 log_to_monitors {default=true} 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 Processor -- start 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 Processor -- start 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 Processor -- start 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 Processor -- start 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 Processor -- start 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 Processor -- start 2019-12-07T19:33:36.551+0000 7f2b7ace7700 10 osd.6 0 agent_entry start 2019-12-07T19:33:36.551+0000 7f2b7ace7700 20 osd.6 0 agent_entry empty queue 2019-12-07T19:33:36.551+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 --2- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] >> v2:172.21.15.131:3300/0 conn(0x5557e6f87800 0x5557e62b4b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.551+0000 7f2b724d6700 5 osd.6 0 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:36.551+0000 7f2b724d6700 20 osd.6 0 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 --2- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] >> v2:172.21.15.109:3300/0 conn(0x5557e6f87000 0x5557e62b5080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] --> v1:172.21.15.109:6789/0 -- auth(proto 0 26 bytes epoch 0) v1 -- 0x5557e6e8c000 con 0x5557e6f87400 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] --> v2:172.21.15.109:3300/0 -- mon_getmap magic: 0 v1 -- 0x5557e627ee00 con 0x5557e6f87000 2019-12-07T19:33:36.551+0000 7f2b9bef3bc0 1 -- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] --> v2:172.21.15.131:3300/0 -- mon_getmap magic: 0 v1 -- 0x5557e627efc0 con 0x5557e6f87800 2019-12-07T19:33:36.551+0000 7f2b967b4700 1 --2- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] >> v2:172.21.15.109:3300/0 conn(0x5557e6f87000 0x5557e62b5080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.551+0000 7f2b957b2700 1 --1- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] >> v1:172.21.15.109:6789/0 conn(0x5557e6f87400 0x5557e70a7000 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=1).handle_server_banner_and_identify peer v1:172.21.15.109:6789/0 says I am v1:172.21.15.109:40576/0 (socket says 172.21.15.109:40576) 2019-12-07T19:33:36.551+0000 7f2b95fb3700 1 --2- [v2:0.0.0.0:6816/36937,v1:0.0.0.0:6817/36937] >> v2:172.21.15.131:3300/0 conn(0x5557e6f87800 0x5557e62b4b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.551+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] learned_addr learned my addr [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] (peer_addr_for_me v1:172.21.15.109:0/0) 2019-12-07T19:33:36.552+0000 7f2b957b2700 10 osd.6 0 new session (outgoing) 0x5557e6295380 con=0x5557e62f5000 addr=v2:172.21.15.109:6816/36937 2019-12-07T19:33:36.552+0000 7f2b88017700 10 osd.6 0 ms_handle_connect con 0x5557e6f87400 2019-12-07T19:33:36.552+0000 7f2b88017700 10 osd.6 0 ms_handle_connect on mon 2019-12-07T19:33:36.552+0000 7f2b88017700 10 osd.6 0 requeue_pg_temp 0 + 0 -> 0 2019-12-07T19:33:36.552+0000 7f2b88017700 20 osd.6 0 _send_ready_to_merge ready_to_merge_source {} not_ready_to_merge_source ready_to_merge_target {} not_ready_to_merge_target {} sent_ready_to_merge_source 2019-12-07T19:33:36.552+0000 7f2b88017700 20 osd.6 0 send_pg_created 2019-12-07T19:33:36.552+0000 7f2b88017700 10 osd.6 0 requeue_failures 0 + 0 -> 0 2019-12-07T19:33:36.552+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> v2:172.21.15.131:3300/0 conn(0x5557e6f87800 msgr2=0x5557e62b4b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:36.552+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> v2:172.21.15.131:3300/0 conn(0x5557e6f87800 0x5557e62b4b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:36.552+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> v1:172.21.15.109:6789/0 conn(0x5557e6f87400 legacy=0x5557e70a7000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:36.552+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> v2:172.21.15.109:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5557e70e3800 con 0x5557e6f87000 2019-12-07T19:33:36.552+0000 7f2b88017700 10 osd.6 0 ms_handle_connect con 0x5557e6f87000 2019-12-07T19:33:36.552+0000 7f2b88017700 10 osd.6 0 ms_handle_connect on mon 2019-12-07T19:33:36.552+0000 7f2b88017700 10 osd.6 0 requeue_pg_temp 0 + 0 -> 0 2019-12-07T19:33:36.552+0000 7f2b88017700 20 osd.6 0 _send_ready_to_merge ready_to_merge_source {} not_ready_to_merge_source ready_to_merge_target {} not_ready_to_merge_target {} sent_ready_to_merge_source 2019-12-07T19:33:36.552+0000 7f2b88017700 20 osd.6 0 send_pg_created 2019-12-07T19:33:36.552+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] conn(0x5557e6f87000 0x5557e62b5080 crc :-1 s=READY pgs=54 cs=0 l=1 rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:36.552+0000 7f2b88017700 10 osd.6 0 requeue_failures 0 + 0 -> 0 2019-12-07T19:33:36.552+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 1 ==== mon_map magic: 0 v1 ==== 383+0+0 (crc 0 0 0) 0x5557e626ee00 con 0x5557e6f87000 2019-12-07T19:33:36.552+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- auth(proto 2 2 bytes epoch 0) v1 -- 0x5557e6289440 con 0x5557e6f87000 2019-12-07T19:33:36.552+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (crc 0 0 0) 0x5557e6281860 con 0x5557e6f87000 2019-12-07T19:33:36.552+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 3 ==== mon_map magic: 0 v1 ==== 383+0+0 (crc 0 0 0) 0x5557e70e3800 con 0x5557e6f87000 2019-12-07T19:33:36.552+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (crc 0 0 0) 0x5557e6289440 con 0x5557e6f87000 2019-12-07T19:33:36.552+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:36.552+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:36.552+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) collect_metadata 2019-12-07T19:33:36.552+0000 7f2b9bef3bc0 10 bluestore(/var/lib/ceph/osd/ceph-6) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:36.552+0000 7f2b9bef3bc0 1 bluestore(/var/lib/ceph/osd/ceph-6) collect_metadata devices span numa nodes 0 2019-12-07T19:33:36.552+0000 7f2b9bef3bc0 10 osd.6 0 mon_cmd_maybe_osd_create cmd: {"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} 2019-12-07T19:33:36.552+0000 7f2b9bef3bc0 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_command({"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} v 0) v1 -- 0x5557e70e3a00 con 0x5557e6f87000 2019-12-07T19:33:36.595+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.645+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.695+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.745+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.795+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.845+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.896+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.946+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.996+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.045+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.096+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.146+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.196+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.246+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.297+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.347+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.397+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.447+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.497+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845356821 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.513+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 5 ==== mon_command_ack([{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]=0 set osd(s) 6 to class 'ssd' v11) v1 ==== 131+0+0 (crc 0 0 0) 0x5557e79b8a00 con 0x5557e6f87000 2019-12-07T19:33:37.513+0000 7f2b9bef3bc0 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:37.513+0000 7f2b9bef3bc0 10 osd.6 0 update_crush_location crush location is "host=smithi109", "root=default" 2019-12-07T19:33:37.513+0000 7f2b9bef3bc0 10 osd.6 0 mon_cmd_maybe_osd_create cmd: {"prefix": "osd crush create-or-move", "id": 6, "weight":0.0879, "args": ["host=smithi109", "root=default"]} 2019-12-07T19:33:37.513+0000 7f2b9bef3bc0 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_command({"prefix": "osd crush create-or-move", "id": 6, "weight":0.0879, "args": ["host=smithi109", "root=default"]} v 0) v1 -- 0x5557e70e3c00 con 0x5557e6f87000 2019-12-07T19:33:37.533+0000 7f2b9171a700 10 osd.6 0 tick_without_osd_lock 2019-12-07T19:33:37.533+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:37.547+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 osd.6 0 tick 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 osd.6 0 do_waiters -- start 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 osd.6 0 do_waiters -- finish 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 osd.6 0 tick last_purged_snaps_scrub 0.000000 next 60.000000 ... now 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 osd.6 0 scrub_purged_snaps 2019-12-07T19:33:37.564+0000 7f2b92fad700 0 log_channel(cluster) log [DBG] : purged_snaps scrub starts 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 snap_mapper.run 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore.onode(0x5557e70ce500).flush flush done 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_omap_iterator has_omap = 0 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore.onode(0x5557e70ce280).flush flush done 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_omap_iterator has_omap = 0 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 snap_mapper.run end, found 0 stray 2019-12-07T19:33:37.564+0000 7f2b92fad700 0 log_channel(cluster) log [DBG] : purged_snaps scrub ok 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 osd.6 0 scrub_purged_snaps done queueing pgs, updating superblock 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 osd.6 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e0 [0,0] lci=[0,0]) 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7046f00 seq 4 2019-12-07T19:33:37.564+0000 7f2b92fad700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7046f00 0 blobs 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 4803 (1 ios * 4000 + 803 bytes) 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes 0x5557e62a1b80 shared_blobs 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:37.564+0000 7f2b92fad700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:37.564+0000 7f2b92fad700 10 osd.6 0 scrub_purged_snaps done 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4270 < min_flush_size 524288 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399535 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4293 < min_flush_size 524288 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4293 < min_flush_size 524288 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x121d~10c5 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x121d 0x121d~10c5 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x121d 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x21d 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x2e2 and padding block with 0xd1e 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x22e2 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:37.564+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000112770s (0.000000076s flush + 0.000112694s kv commit) 2019-12-07T19:33:37.564+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:37.564+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:37.564+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:37.564+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:37.564+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:37.564+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:37.564+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7046f00 osr 0x5557e76ce500 2019-12-07T19:33:37.564+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e76cf2c0) _discard drop 1 0x2000~1000 2019-12-07T19:33:37.564+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e76cf2c0) prepare_write seq 2 0x2000~1000 crc 96158729 2019-12-07T19:33:37.564+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:37.597+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.647+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.698+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.748+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.798+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.848+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.898+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.948+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.998+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.048+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.099+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.149+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.199+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.249+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.299+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.349+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.399+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.449+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.500+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415075 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.514+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 6 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0879, "args": ["host=smithi109", "root=default"]}]=0 create-or-move updating item name 'osd.6' weight 0.0879 at location {host=smithi109,root=default} to crush map v12) v1 ==== 252+0+0 (crc 0 0 0) 0x5557e79b8800 con 0x5557e6f87000 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6 0 ensuring pgs have consumed prior maps 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 7 osd.6 0 consume_map version 0 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 20 osd.6 0 prune_pg_created 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6:0.consume_map 0 (was 0) 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6:1.consume_map 0 (was 0) 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6:2.consume_map 0 (was 0) 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6:3.consume_map 0 (was 0) 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6:4.consume_map 0 (was 0) 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6:5.consume_map 0 (was 0) 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6:6.consume_map 0 (was 0) 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6:7.consume_map 0 (was 0) 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6 0 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 0 osd.6 0 done with init, starting boot process 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({mgrmap=0+,osd_pg_creates=0+}) v3 -- 0x5557e70e3e00 con 0x5557e6f87000 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 1 osd.6 0 start_boot 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 10 osd.6 0 start_boot - have maps 0..0 2019-12-07T19:33:38.514+0000 7f2b9bef3bc0 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x5557e627f340 con 0x5557e6f87000 2019-12-07T19:33:38.517+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 7 ==== mgrmap(e 4) v1 ==== 45131+0+0 (crc 0 0 0) 0x5557e7121500 con 0x5557e6f87000 2019-12-07T19:33:38.517+0000 7f2b88017700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:38.517+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:38.519+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.6) v3 -- 0x5557e6e8f600 con 0x5557e7a48000 2019-12-07T19:33:38.519+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 crc :-1 s=READY pgs=21 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:38.519+0000 7f2b88017700 10 osd.6 0 ms_handle_connect con 0x5557e7a48000 2019-12-07T19:33:38.519+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 8 ==== mon_get_version_reply(handle=1 version=12) v2 ==== 24+0+0 (crc 0 0 0) 0x5557e6281680 con 0x5557e6f87000 2019-12-07T19:33:38.519+0000 7f2b844fa700 10 osd.6 0 _preboot _preboot mon has osdmaps 1..12 2019-12-07T19:33:38.519+0000 7f2b844fa700 5 osd.6 0 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:38.519+0000 7f2b844fa700 20 osd.6 0 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:38.519+0000 7f2b844fa700 -1 osd.6 0 waiting for initial osdmap 2019-12-07T19:33:38.519+0000 7f2b844fa700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=1}) v3 -- 0x5557e79b9800 con 0x5557e6f87000 2019-12-07T19:33:38.519+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 msgr2=0x5557e62b5600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:33:38.519+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 msgr2=0x5557e62b5600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:38.519+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 crc :-1 s=READY pgs=21 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:38.519+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 crc :-1 s=READY pgs=21 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:38.519+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 msgr2=0x5557e62b5600 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:33:38.519+0000 7f2b88017700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:38.521+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 9 ==== osd_map(1..12 src has 1..12) v4 ==== 5778+0+0 (crc 0 0 0) 0x5557e799ad80 con 0x5557e6f87000 2019-12-07T19:33:38.521+0000 7f2b88017700 20 osd.6 0 OSD::ms_dispatch: osd_map(1..12 src has 1..12) v4 2019-12-07T19:33:38.521+0000 7f2b88017700 10 osd.6 0 do_waiters -- start 2019-12-07T19:33:38.521+0000 7f2b88017700 10 osd.6 0 do_waiters -- finish 2019-12-07T19:33:38.521+0000 7f2b88017700 20 osd.6 0 _dispatch 0x5557e799ad80 osd_map(1..12 src has 1..12) v4 2019-12-07T19:33:38.521+0000 7f2b88017700 3 osd.6 0 handle_osd_map epochs [1,12], i have 0, src has [1,12] 2019-12-07T19:33:38.521+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 1 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 1, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 2 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:8d9276bf:::osdmap.1:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:8d9276bf:::osdmap.1:0# key 0x7f7fffffffffffffff8d9276bf216f73'dmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:8d9276bf:::osdmap.1:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 2, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 3 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:829276bf:::osdmap.2:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:829276bf:::osdmap.2:0# key 0x7f7fffffffffffffff829276bf216f73'dmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:829276bf:::osdmap.2:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 3, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 4 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:887276bf:::osdmap.3:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:887276bf:::osdmap.3:0# key 0x7f7fffffffffffffff887276bf216f73'dmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:887276bf:::osdmap.3:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 4, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 5 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:85f276bf:::osdmap.4:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:85f276bf:::osdmap.4:0# key 0x7f7fffffffffffffff85f276bf216f73'dmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:85f276bf:::osdmap.4:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 5, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 6 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:8ef276bf:::osdmap.5:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:8ef276bf:::osdmap.5:0# key 0x7f7fffffffffffffff8ef276bf216f73'dmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:8ef276bf:::osdmap.5:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 6, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 7 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:803276bf:::osdmap.6:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:803276bf:::osdmap.6:0# key 0x7f7fffffffffffffff803276bf216f73'dmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:803276bf:::osdmap.6:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 7, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 8 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:8bb276bf:::osdmap.7:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:8bb276bf:::osdmap.7:0# key 0x7f7fffffffffffffff8bb276bf216f73'dmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:8bb276bf:::osdmap.7:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 8, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 9 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:86b276bf:::osdmap.8:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:86b276bf:::osdmap.8:0# key 0x7f7fffffffffffffff86b276bf216f73'dmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:86b276bf:::osdmap.8:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 9, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 10 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:8c4276bf:::osdmap.9:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:8c4276bf:::osdmap.9:0# key 0x7f7fffffffffffffff8c4276bf216f73'dmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:8c4276bf:::osdmap.9:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 10, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 11 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3890e826:::osdmap.10:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3890e826:::osdmap.10:0# key 0x7f7fffffffffffffff3890e8'&!osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3890e826:::osdmap.10:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 11, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 handle_osd_map got inc map for epoch 12 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3570e826:::osdmap.11:0# 0x0~0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3570e826:::osdmap.11:0# key 0x7f7fffffffffffffff3570e8'&!osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3570e826:::osdmap.11:0# 0x0~0 = -2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 got_full_map 12, nothing requested 2019-12-07T19:33:38.522+0000 7f2b88017700 20 osd.6 0 handle_osd_map pg_num_history pg_num_history(e12 pg_nums {} deleted_pools ) 2019-12-07T19:33:38.522+0000 7f2b88017700 10 snap_mapper.record_purged_snaps purged_snaps {1={},2={},3={},4={},5={},6={},7={},8={},9={},10={},11={},12={}} 2019-12-07T19:33:38.522+0000 7f2b88017700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:38.522+0000 7f2b88017700 10 osd.6 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e12 [1,12] lci=[0,0]) 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7121500 seq 5 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:60c2fa6d:::inc_osdmap.1:0# key 0x7f7fffffffffffffff60c2fa'm!inc_osdmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1028 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~3b2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xc4e on front/back, now 0x0~1000 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.522+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x3000~1000/1000,1000,0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x3000~1000] 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a90000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a90000 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90070 sbid 0x0)) 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90000) get_ref 0x0~3b2 Blob(0x5557e7a90000 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90070 sbid 0x0)) 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90000) get_ref init 0x1000, 1000 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~3b2: 0x0~3b2 Blob(0x5557e7a90000 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3b2) SharedBlob(0x5557e7a90070 sbid 0x0)) 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90088 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x3b2 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf288) dirty_range mark inline shard dirty 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 = 0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:8d9276bf:::osdmap.1:0# key 0x7f7fffffffffffffff8d9276bf216f73'dmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1029 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~2c6 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xd3a on front/back, now 0x0~1000 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.522+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.522+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x4000~1000/1000,1000,0 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x4000~1000] 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a29f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a29f80 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29ea0 sbid 0x0)) 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29f80) get_ref 0x0~2c6 Blob(0x5557e7a29f80 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29ea0 sbid 0x0)) 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29f80) get_ref init 0x1000, 1000 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2c6: 0x0~2c6 Blob(0x5557e7a29f80 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x2c6) SharedBlob(0x5557e7a29ea0 sbid 0x0)) 2019-12-07T19:33:38.522+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a29eb8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x2c6 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf508) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:6b02fa6d:::inc_osdmap.2:0# key 0x7f7fffffffffffffff6b02fa'm!inc_osdmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1030 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24c 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xdb4 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x5000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x5000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a90540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a904d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a90540 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a904d0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90540) get_ref 0x0~24c Blob(0x5557e7a90540 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a904d0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90540) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~24c: 0x0~24c Blob(0x5557e7a90540 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24c) SharedBlob(0x5557e7a904d0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a904e8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x24c 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf788) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:829276bf:::osdmap.2:0# key 0x7f7fffffffffffffff829276bf216f73'dmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:829276bf:::osdmap.2:0# 0x0~2c6 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1031 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:829276bf:::osdmap.2:0# 0x0~2c6 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~2c6 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xd3a on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x6000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x6000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a903f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a903f0 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90380 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a903f0) get_ref 0x0~2c6 Blob(0x5557e7a903f0 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90380 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a903f0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2c6: 0x0~2c6 Blob(0x5557e7a903f0 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x2c6) SharedBlob(0x5557e7a90380 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90398 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x2c6 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cfa08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:829276bf:::osdmap.2:0# 0x0~2c6 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:6602fa6d:::inc_osdmap.3:0# key 0x7f7fffffffffffffff6602fa'm!inc_osdmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1032 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x7000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x7000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a902a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a902a0 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90230 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a902a0) get_ref 0x0~108 Blob(0x5557e7a902a0 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90230 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a902a0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x5557e7a902a0 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x5557e7a90230 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90248 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cfc88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:887276bf:::osdmap.3:0# key 0x7f7fffffffffffffff887276bf216f73'dmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:887276bf:::osdmap.3:0# 0x0~339 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1033 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:887276bf:::osdmap.3:0# 0x0~339 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~339 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xcc7 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x8000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x8000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a29dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a29dc0 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29d50 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29dc0) get_ref 0x0~339 Blob(0x5557e7a29dc0 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29d50 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29dc0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~339: 0x0~339 Blob(0x5557e7a29dc0 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x339) SharedBlob(0x5557e7a29d50 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a29d68 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x339 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee108) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:887276bf:::osdmap.3:0# 0x0~339 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:6c82fa6d:::inc_osdmap.4:0# key 0x7f7fffffffffffffff6c82fa'm!inc_osdmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1034 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x9000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x9000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a29730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a29730 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29f10 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29730) get_ref 0x0~108 Blob(0x5557e7a29730 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29f10 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29730) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x5557e7a29730 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x5557e7a29f10 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a29f28 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee388) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:85f276bf:::osdmap.4:0# key 0x7f7fffffffffffffff85f276bf216f73'dmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:85f276bf:::osdmap.4:0# 0x0~3ac 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1035 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:85f276bf:::osdmap.4:0# 0x0~3ac - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~3ac 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xc54 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xa000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xa000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a90690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a90690 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90700 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90690) get_ref 0x0~3ac Blob(0x5557e7a90690 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90700 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90690) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~3ac: 0x0~3ac Blob(0x5557e7a90690 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3ac) SharedBlob(0x5557e7a90700 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x3ac 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee608) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:85f276bf:::osdmap.4:0# 0x0~3ac = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:6362fa6d:::inc_osdmap.5:0# key 0x7f7fffffffffffffff6362fa'm!inc_osdmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1036 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xb000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xb000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a907e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a907e0 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90850 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a907e0) get_ref 0x0~108 Blob(0x5557e7a907e0 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90850 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a907e0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x5557e7a907e0 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x5557e7a90850 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90868 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee888) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:8ef276bf:::osdmap.5:0# key 0x7f7fffffffffffffff8ef276bf216f73'dmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:8ef276bf:::osdmap.5:0# 0x0~41f 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1037 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:8ef276bf:::osdmap.5:0# 0x0~41f - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~41f 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xbe1 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xc000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xc000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a90930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a909a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a90930 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a909a0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90930) get_ref 0x0~41f Blob(0x5557e7a90930 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a909a0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90930) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~41f: 0x0~41f Blob(0x5557e7a90930 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x41f) SharedBlob(0x5557e7a909a0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a909b8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x41f 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79eeb08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:8ef276bf:::osdmap.5:0# 0x0~41f = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:69e2fa6d:::inc_osdmap.6:0# key 0x7f7fffffffffffffff69e2fa'm!inc_osdmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1038 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xd000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xd000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a90a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a90a80 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90af0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90a80) get_ref 0x0~108 Blob(0x5557e7a90a80 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90af0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90a80) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x5557e7a90a80 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x5557e7a90af0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90b08 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79eed88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:803276bf:::osdmap.6:0# key 0x7f7fffffffffffffff803276bf216f73'dmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:803276bf:::osdmap.6:0# 0x0~492 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1039 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:803276bf:::osdmap.6:0# 0x0~492 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~492 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xb6e on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xe000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xe000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a90bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a90bd0 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90c40 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90bd0) get_ref 0x0~492 Blob(0x5557e7a90bd0 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90c40 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90bd0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~492: 0x0~492 Blob(0x5557e7a90bd0 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x492) SharedBlob(0x5557e7a90c40 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90c58 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x492 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef008) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:803276bf:::osdmap.6:0# 0x0~492 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:64e2fa6d:::inc_osdmap.7:0# key 0x7f7fffffffffffffff64e2fa'm!inc_osdmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1040 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xf000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xf000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a29e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a29e30 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90d20 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29e30) get_ref 0x0~108 Blob(0x5557e7a29e30 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90d20 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29e30) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x5557e7a29e30 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x5557e7a90d20 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90d38 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef288) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:8bb276bf:::osdmap.7:0# key 0x7f7fffffffffffffff8bb276bf216f73'dmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:8bb276bf:::osdmap.7:0# 0x0~505 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1041 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:8bb276bf:::osdmap.7:0# 0x0~505 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~505 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xafb on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x10000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x10000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a90e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a90e00 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90e70 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90e00) get_ref 0x0~505 Blob(0x5557e7a90e00 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90e70 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90e00) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~505: 0x0~505 Blob(0x5557e7a90e00 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x505) SharedBlob(0x5557e7a90e70 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90e88 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x505 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef508) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:8bb276bf:::osdmap.7:0# 0x0~505 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:6f22fa6d:::inc_osdmap.8:0# key 0x7f7fffffffffffffff6f22fa'm!inc_osdmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1042 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x11000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x11000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a90f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a90f50 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90fc0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90f50) get_ref 0x0~108 Blob(0x5557e7a90f50 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a90fc0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a90f50) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x5557e7a90f50 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x5557e7a90fc0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a90fd8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef788) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:86b276bf:::osdmap.8:0# key 0x7f7fffffffffffffff86b276bf216f73'dmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:86b276bf:::osdmap.8:0# 0x0~578 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1043 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:86b276bf:::osdmap.8:0# 0x0~578 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~578 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xa88 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x12000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x12000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a910a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a910a0 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91110 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a910a0) get_ref 0x0~578 Blob(0x5557e7a910a0 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91110 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a910a0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~578: 0x0~578 Blob(0x5557e7a910a0 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x578) SharedBlob(0x5557e7a91110 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91128 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x578 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79efa08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:86b276bf:::osdmap.8:0# 0x0~578 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:61a2fa6d:::inc_osdmap.9:0# key 0x7f7fffffffffffffff61a2fa'm!inc_osdmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1044 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x13000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x13000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a911f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a911f0 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91260 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a911f0) get_ref 0x0~108 Blob(0x5557e7a911f0 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91260 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a911f0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x5557e7a911f0 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x5557e7a91260 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91278 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79efc88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:8c4276bf:::osdmap.9:0# key 0x7f7fffffffffffffff8c4276bf216f73'dmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:8c4276bf:::osdmap.9:0# 0x0~5eb 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1045 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:8c4276bf:::osdmap.9:0# 0x0~5eb - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~5eb 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xa15 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x14000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x14000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a91340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a913b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a91340 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a913b0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91340) get_ref 0x0~5eb Blob(0x5557e7a91340 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a913b0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91340) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~5eb: 0x0~5eb Blob(0x5557e7a91340 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x5eb) SharedBlob(0x5557e7a913b0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a913c8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x5eb 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6108) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:8c4276bf:::osdmap.9:0# 0x0~5eb = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c003972f:::inc_osdmap.10:0# key 0x7f7fffffffffffffffc00397'/!inc_osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c003972f:::inc_osdmap.10:0# 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1046 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c003972f:::inc_osdmap.10:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x15000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x15000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a91490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a91490 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91500 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91490) get_ref 0x0~108 Blob(0x5557e7a91490 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91500 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91490) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x5557e7a91490 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x5557e7a91500 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91518 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x108 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6388) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c003972f:::inc_osdmap.10:0# 0x0~108 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3890e826:::osdmap.10:0# key 0x7f7fffffffffffffff3890e8'&!osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3890e826:::osdmap.10:0# 0x0~65e 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1047 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3890e826:::osdmap.10:0# 0x0~65e - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~65e 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0x9a2 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x16000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x16000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a915e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a915e0 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91650 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a915e0) get_ref 0x0~65e Blob(0x5557e7a915e0 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91650 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a915e0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~65e: 0x0~65e Blob(0x5557e7a915e0 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x65e) SharedBlob(0x5557e7a91650 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91668 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x65e 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6608) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3890e826:::osdmap.10:0# 0x0~65e = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:cb83972f:::inc_osdmap.11:0# key 0x7f7fffffffffffffffcb8397'/!inc_osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1048 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~33a 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xcc6 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x17000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x17000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a91730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a917a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a91730 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a917a0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91730) get_ref 0x0~33a Blob(0x5557e7a91730 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a917a0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91730) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~33a: 0x0~33a Blob(0x5557e7a91730 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x33a) SharedBlob(0x5557e7a917a0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a917b8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x33a 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6888) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3570e826:::osdmap.11:0# key 0x7f7fffffffffffffff3570e8'&!osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3570e826:::osdmap.11:0# 0x0~74c 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1049 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3570e826:::osdmap.11:0# 0x0~74c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~74c 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0x8b4 on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x18000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x18000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a91880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a918f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a91880 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a918f0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91880) get_ref 0x0~74c Blob(0x5557e7a91880 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a918f0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91880) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~74c: 0x0~74c Blob(0x5557e7a91880 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x74c) SharedBlob(0x5557e7a918f0 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91908 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x74c 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6b08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3570e826:::osdmap.11:0# 0x0~74c = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c683972f:::inc_osdmap.12:0# key 0x7f7fffffffffffffffc68397'/!inc_osdmap.12!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c683972f:::inc_osdmap.12:0# 0x0~496 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1050 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c683972f:::inc_osdmap.12:0# 0x0~496 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~496 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xb6a on front/back, now 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x19000~1000/1000,1000,0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x19000~1000] 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a919d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a919d0 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91a40 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a919d0) get_ref 0x0~496 Blob(0x5557e7a919d0 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91a40 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.blob(0x5557e7a919d0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~496: 0x0~496 Blob(0x5557e7a919d0 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x496) SharedBlob(0x5557e7a91a40 sbid 0x0)) 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91a58 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x496 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6d88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.523+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c683972f:::inc_osdmap.12:0# 0x0~496 = 0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3e70e826:::osdmap.12:0# key 0x7f7fffffffffffffff3e70e8'&!osdmap.12!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.523+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1051 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3e70e826:::osdmap.12:0# 0x0~8a8 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.523+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~8a8 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0x758 on front/back, now 0x0~1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.524+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.524+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1a000~1000/1000,1000,0 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x1a000~1000] 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a91b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a91b20 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91b90 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91b20) get_ref 0x0~8a8 Blob(0x5557e7a91b20 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91b90 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91b20) get_ref init 0x1000, 1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~8a8: 0x0~8a8 Blob(0x5557e7a91b20 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x8a8) SharedBlob(0x5557e7a91b90 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91ba8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x8a8 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7008) dirty_range mark inline shard dirty 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 = 0 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:46e858f0:::pg_num_history:head# key 0x7f7fffffffffffffff46e858f0217067'_num_history!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:38.524+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1052 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~12 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xfee on front/back, now 0x0~1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:38.524+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.524+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1b000~1000/1000,1000,0 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x1b000~1000] 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a91c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) get_ref init 0x1000, 1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x12 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:38.524+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:38.524+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:38.524+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 0 blobs 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121500 cost 129493 (26 ios * 4000 + 25493 bytes) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121500 onodes 0x5557e62a1b80,0x5557e70c6000,0x5557e70c6280,0x5557e70c6500,0x5557e70c6780,0x5557e70c6a00,0x5557e70c6c80,0x5557e70c6f00,0x5557e70c7180,0x5557e70ce500,0x5557e70cf180,0x5557e70cf400,0x5557e70cf680,0x5557e70cf900,0x5557e70cfb80,0x5557e79ee000,0x5557e79ee280,0x5557e79ee500,0x5557e79ee780,0x5557e79eea00,0x5557e79eec80,0x5557e79eef00,0x5557e79ef180,0x5557e79ef400,0x5557e79ef680,0x5557e79ef900,0x5557e79efb80 shared_blobs 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6108) update #-1:8c4276bf:::osdmap.9:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6108) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:8c4276bf:::osdmap.9:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6388) update #-1:c003972f:::inc_osdmap.10:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6388) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c003972f:::inc_osdmap.10:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6608) update #-1:3890e826:::osdmap.10:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6608) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3890e826:::osdmap.10:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6888) update #-1:cb83972f:::inc_osdmap.11:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6888) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:cb83972f:::inc_osdmap.11:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6b08) update #-1:3570e826:::osdmap.11:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6b08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3570e826:::osdmap.11:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6d88) update #-1:c683972f:::inc_osdmap.12:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c6d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c683972f:::inc_osdmap.12:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7008) update #-1:3e70e826:::osdmap.12:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3e70e826:::osdmap.12:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70ce608) update #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c1a3fc6e:::purged_snaps:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf288) update #-1:60c2fa6d:::inc_osdmap.1:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:60c2fa6d:::inc_osdmap.1:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf508) update #-1:8d9276bf:::osdmap.1:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:8d9276bf:::osdmap.1:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf788) update #-1:6b02fa6d:::inc_osdmap.2:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cf788) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:6b02fa6d:::inc_osdmap.2:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cfa08) update #-1:829276bf:::osdmap.2:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cfa08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:829276bf:::osdmap.2:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cfc88) update #-1:6602fa6d:::inc_osdmap.3:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70cfc88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:6602fa6d:::inc_osdmap.3:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee108) update #-1:887276bf:::osdmap.3:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee108) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:887276bf:::osdmap.3:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee388) update #-1:6c82fa6d:::inc_osdmap.4:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee388) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:6c82fa6d:::inc_osdmap.4:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee608) update #-1:85f276bf:::osdmap.4:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee608) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:85f276bf:::osdmap.4:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee888) update #-1:6362fa6d:::inc_osdmap.5:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ee888) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:6362fa6d:::inc_osdmap.5:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79eeb08) update #-1:8ef276bf:::osdmap.5:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79eeb08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:8ef276bf:::osdmap.5:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79eed88) update #-1:69e2fa6d:::inc_osdmap.6:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79eed88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:69e2fa6d:::inc_osdmap.6:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef008) update #-1:803276bf:::osdmap.6:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:803276bf:::osdmap.6:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef288) update #-1:64e2fa6d:::inc_osdmap.7:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:64e2fa6d:::inc_osdmap.7:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef508) update #-1:8bb276bf:::osdmap.7:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:8bb276bf:::osdmap.7:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef788) update #-1:6f22fa6d:::inc_osdmap.8:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79ef788) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:6f22fa6d:::inc_osdmap.8:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79efa08) update #-1:86b276bf:::osdmap.8:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79efa08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:86b276bf:::osdmap.8:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79efc88) update #-1:61a2fa6d:::inc_osdmap.9:0# 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore.extentmap(0x5557e79efc88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:61a2fa6d:::inc_osdmap.9:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.524+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121500 allocated 0x[3000~19000] released 0x[] 2019-12-07T19:33:38.524+0000 7f2b88017700 10 freelist allocate 0x3000~19000 2019-12-07T19:33:38.524+0000 7f2b88017700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 prepare 2019-12-07T19:33:38.524+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7121500 2019-12-07T19:33:38.524+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 aio_wait 2019-12-07T19:33:38.524+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121500 2019-12-07T19:33:38.524+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 io_done 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6836 < min_flush_size 524288 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c6000 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c6280 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c6500 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c6780 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c6a00 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c6c80 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c6f00 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70cf180 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70cf400 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70cf680 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70cf900 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70cfb80 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79ee000 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79ee280 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79ee500 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79ee780 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79eea00 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79eec80 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79eef00 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79ef180 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79ef400 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79ef680 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79ef900 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e79efb80 had 1 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6859 < min_flush_size 524288 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6859 < min_flush_size 524288 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x22e2~1acb to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x22e2 0x22e2~1acb to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x22e2 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x2e2 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xdad and padding block with 0x253 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3dad 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:38.524+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000188745s (0.000050414s flush + 0.000138331s kv commit) 2019-12-07T19:33:38.524+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:38.524+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:38.524+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121500> 2019-12-07T19:33:38.524+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:38.524+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:38.524+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121500 2019-12-07T19:33:38.524+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7121500 osr 0x5557e76ce500 2019-12-07T19:33:38.524+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e76cf2c0) _discard drop 2 0x2000~1000 2019-12-07T19:33:38.524+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e76cf2c0) prepare_write seq 3 0x2000~1000 crc df944ae1 2019-12-07T19:33:38.524+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 0 _committed_osd_maps 1..12 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 0 advance to epoch 1 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 1 advance to epoch 2 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 2 advance to epoch 3 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 3 advance to epoch 4 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 4 advance to epoch 5 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 5 advance to epoch 6 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 6 advance to epoch 7 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 7 advance to epoch 8 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 8 advance to epoch 9 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 9 advance to epoch 10 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 10 advance to epoch 11 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 osd.6 11 advance to epoch 12 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 0 osd.6 12 crush map has features 288514050185494528, adjusting msgr requires for clients 2019-12-07T19:33:38.524+0000 7f2b85cfd700 0 osd.6 12 crush map has features 288514050185494528 was 288232575208792577, adjusting msgr requires for mons 2019-12-07T19:33:38.524+0000 7f2b85cfd700 0 osd.6 12 crush map has features 3314932999778484224, adjusting msgr requires for osds 2019-12-07T19:33:38.524+0000 7f2b85cfd700 1 osd.6 12 check_osdmap_features require_osd_release unknown -> octopus 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label got bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 7 meta) 2019-12-07T19:33:38.524+0000 7f2b85cfd700 10 bluestore(/var/lib/ceph/osd/ceph-6/block) _write_bdev_label path /var/lib/ceph/osd/ceph-6/block label bdev(osd_uuid 429b1431-873f-4985-95c1-51e7035621d2, size 0x1680000000, btime 2019-12-07T19:33:18.406885+0000, desc main, 8 meta) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 7 osd.6 12 consume_map version 12 2019-12-07T19:33:38.525+0000 7f2b85cfd700 20 osd.6 12 prune_pg_created 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6:0.consume_map 12 (was 0) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6:1.consume_map 12 (was 0) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6:2.consume_map 12 (was 0) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6:3.consume_map 12 (was 0) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6:4.consume_map 12 (was 0) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6:5.consume_map 12 (was 0) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6:6.consume_map 12 (was 0) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6:7.consume_map 12 (was 0) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6 12 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:38.525+0000 7f2b85cfd700 10 osd.6 12 _preboot _preboot mon has osdmaps 1..12 2019-12-07T19:33:38.525+0000 7f2b85cfd700 5 osd.6 12 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:38.525+0000 7f2b85cfd700 20 osd.6 12 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:38.525+0000 7f2b8ff17700 10 osd.6 12 operator() waiting for peering work to drain 2019-12-07T19:33:38.525+0000 7f2b8ff17700 10 osd.6 12 _send_boot 2019-12-07T19:33:38.525+0000 7f2b8ff17700 20 osd.6 12 initial client_addrs [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937], cluster_addrs [v2:0.0.0.0:6818/36937,v1:0.0.0.0:6819/36937], hb_back_addrs [v2:0.0.0.0:6822/36937,v1:0.0.0.0:6823/36937], hb_front_addrs [v2:0.0.0.0:6820/36937,v1:0.0.0.0:6821/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6818/36937,v1:0.0.0.0:6819/36937] set_addr_unknowns [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6818/36937,v1:0.0.0.0:6819/36937] set_addr_unknowns assuming my addr v2:0.0.0.0:6818/36937 matches provided addr v2:172.21.15.109:6816/36937 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6818/36937,v1:0.0.0.0:6819/36937] set_addr_unknowns assuming my addr v1:0.0.0.0:6819/36937 matches provided addr v2:172.21.15.109:6816/36937 2019-12-07T19:33:38.525+0000 7f2b8ff17700 10 osd.6 12 new session (outgoing) 0x5557e6295a00 con=0x5557e62f5400 addr=v2:172.21.15.109:6818/36937 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] set_addr_unknowns now [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 10 osd.6 12 assuming cluster_addrs match client_addrs [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6822/36937,v1:0.0.0.0:6823/36937] set_addr_unknowns [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6822/36937,v1:0.0.0.0:6823/36937] set_addr_unknowns assuming my addr v2:0.0.0.0:6822/36937 matches provided addr v2:172.21.15.109:6818/36937 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6822/36937,v1:0.0.0.0:6823/36937] set_addr_unknowns assuming my addr v1:0.0.0.0:6823/36937 matches provided addr v2:172.21.15.109:6818/36937 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] set_addr_unknowns now [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 10 osd.6 12 assuming hb_back_addrs match cluster_addrs [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6820/36937,v1:0.0.0.0:6821/36937] set_addr_unknowns [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6820/36937,v1:0.0.0.0:6821/36937] set_addr_unknowns assuming my addr v2:0.0.0.0:6820/36937 matches provided addr v2:172.21.15.109:6816/36937 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:0.0.0.0:6820/36937,v1:0.0.0.0:6821/36937] set_addr_unknowns assuming my addr v1:0.0.0.0:6821/36937 matches provided addr v2:172.21.15.109:6816/36937 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] set_addr_unknowns now [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 10 osd.6 12 assuming hb_front_addrs match client_addrs [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] 2019-12-07T19:33:38.525+0000 7f2b8ff17700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.525+0000 7f2b8ff17700 1 osd.6 12 set_numa_affinity storage numa node 0 2019-12-07T19:33:38.526+0000 7f2b8ff17700 1 osd.6 12 set_numa_affinity public network enp3s0f1 numa node 0 2019-12-07T19:33:38.526+0000 7f2b8ff17700 1 osd.6 12 set_numa_affinity cluster network enp3s0f1 numa node 0 2019-12-07T19:33:38.526+0000 7f2b8ff17700 1 osd.6 12 objectstore and network numa nodes all match 2019-12-07T19:33:38.526+0000 7f2b8ff17700 1 osd.6 12 set_numa_affinity setting numa affinity to node 0 cpus 0-7 2019-12-07T19:33:38.526+0000 7f2b8ff17700 10 osd.6 12 final client_addrs [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937], cluster_addrs [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937], hb_back_addrs [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937], hb_front_addrs [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] 2019-12-07T19:33:38.526+0000 7f2b8ff17700 10 bluestore(/var/lib/ceph/osd/ceph-6) collect_metadata 2019-12-07T19:33:38.527+0000 7f2b8ff17700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.527+0000 7f2b8ff17700 1 bluestore(/var/lib/ceph/osd/ceph-6) collect_metadata devices span numa nodes 0 2019-12-07T19:33:38.527+0000 7f2b8ff17700 10 bluestore(/var/lib/ceph/osd/ceph-6) collect_metadata 2019-12-07T19:33:38.527+0000 7f2b8ff17700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.527+0000 7f2b8ff17700 1 bluestore(/var/lib/ceph/osd/ceph-6) collect_metadata devices span numa nodes 0 2019-12-07T19:33:38.528+0000 7f2b8ff17700 10 osd.6 12 _collect_metadata {arch=x86_64,back_addr=[v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937],back_iface=enp3s0f1,bluefs=1,bluefs_dedicated_db=0,bluefs_dedicated_wal=0,bluefs_single_shared_device=1,bluestore_bdev_access_mode=file,bluestore_bdev_block_size=4096,bluestore_bdev_devices=nvme0n1,bluestore_bdev_driver=KernelDevice,bluestore_bdev_path=/var/lib/ceph/osd/ceph-6/block,bluestore_bdev_rotational=0,bluestore_bdev_size=96636764160,bluestore_bdev_support_discard=1,bluestore_bdev_type=ssd,ceph_release=octopus,ceph_version=ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev),ceph_version_short=15.0.0-8038-g503c226,cpu=Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz,default_device_class=ssd,device_ids=nvme0n1=INTEL_SSDPEDMD400G4_PHFT6204016X400BGN,devices=nvme0n1,distro=rhel,distro_description=Employee SKU,distro_version=7.7,front_addr=[v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937],front_iface=enp3s0f1,hb_back_addr=[v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937],hb_front_addr=[v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937],hostname=smithi109,journal_rotational=0,kernel_description=#1 SMP Wed Sep 25 09:42:57 EDT 2019,kernel_version=3.10.0-1062.4.1.el7.x86_64,mem_swap_kb=0,mem_total_kb=32664132,network_numa_node=0,network_numa_nodes=0,numa_node=0,numa_node_cpus=0-7,objectstore_numa_node=0,objectstore_numa_nodes=0,os=Linux,osd_data=/var/lib/ceph/osd/ceph-6,osd_objectstore=bluestore,rotational=0} 2019-12-07T19:33:38.528+0000 7f2b8ff17700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_boot(osd.6 booted 0 features 4540138292836696063 v12) v7 -- 0x5557e79c4a80 con 0x5557e6f87000 2019-12-07T19:33:38.539+0000 7f2b92fad700 10 osd.6 12 tick 2019-12-07T19:33:38.539+0000 7f2b92fad700 1 osd.6 12 tick checking mon for new map 2019-12-07T19:33:38.539+0000 7f2b92fad700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=13}) v3 -- 0x5557e79b9800 con 0x5557e6f87000 2019-12-07T19:33:38.539+0000 7f2b92fad700 10 osd.6 12 do_waiters -- start 2019-12-07T19:33:38.539+0000 7f2b92fad700 10 osd.6 12 do_waiters -- finish 2019-12-07T19:33:38.539+0000 7f2b92fad700 20 osd.6 12 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:38.550+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.579+0000 7f2b9171a700 10 osd.6 12 tick_without_osd_lock 2019-12-07T19:33:38.579+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffe4000/0x40000000/0x1680000000, data 0x544a/0x1a000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:38.600+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.650+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.700+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.750+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.800+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.850+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.901+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.951+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.001+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.051+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.101+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.151+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.201+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.251+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.302+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.352+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.402+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.452+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.502+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.518+0000 7f2b83cf9700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.518+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.6) v3 -- 0x5557e6e8f8c0 con 0x5557e7a48000 2019-12-07T19:33:39.519+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.519+0000 7f2b88017700 10 osd.6 12 ms_handle_connect con 0x5557e7a48000 2019-12-07T19:33:39.519+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 crc :-1 s=READY pgs=30 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.520+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 msgr2=0x5557e62b5600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:33:39.520+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 msgr2=0x5557e62b5600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:39.520+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mgr.4099 v2:172.21.15.131:6800/36137 1 ==== mgrconfigure(period=5, threshold=5) v3 ==== 12+0+0 (crc 0 0 0) 0x5557e62b8800 con 0x5557e7a48000 2019-12-07T19:33:39.520+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 crc :-1 s=READY pgs=30 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:39.520+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 crc :-1 s=READY pgs=30 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.520+0000 7f2b88017700 10 osd.6 12 setting 0 queries 2019-12-07T19:33:39.520+0000 7f2b88017700 20 osd.6 12 reports for 0 queries 2019-12-07T19:33:39.520+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.6 +109-0 packed 1198 daemon_metrics=2) v8 -- 0x5557e79c4380 con 0x5557e7a48000 2019-12-07T19:33:39.520+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x5557e79c4380 con 0x5557e7a48000 2019-12-07T19:33:39.520+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 msgr2=0x5557e62b5600 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:33:39.520+0000 7f2b88017700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.521+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 10 ==== osd_map(13..13 src has 1..13) v4 ==== 3272+0+0 (crc 0 0 0) 0x5557e70c2fc0 con 0x5557e6f87000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 osd.6 12 OSD::ms_dispatch: osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.521+0000 7f2b88017700 10 osd.6 12 do_waiters -- start 2019-12-07T19:33:39.521+0000 7f2b88017700 10 osd.6 12 do_waiters -- finish 2019-12-07T19:33:39.521+0000 7f2b88017700 20 osd.6 12 _dispatch 0x5557e70c2fc0 osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.521+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5557e62b8800 con 0x5557e6f87000 2019-12-07T19:33:39.521+0000 7f2b88017700 3 osd.6 12 handle_osd_map epochs [13,13], i have 12, src has [1,13] 2019-12-07T19:33:39.521+0000 7f2b88017700 10 osd.6 12 handle_osd_map got inc map for epoch 13 2019-12-07T19:33:39.521+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3e70e826:::osdmap.12:0# 0x0~0 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~8a8 size 0x8a8 (2216) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7a91b20 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x8a8) SharedBlob(0x5557e7a91b90 sbid 0x0)) need 0x0~8a8 cache has 0x[0~8a8] 2019-12-07T19:33:39.521+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 = 2216 2019-12-07T19:33:39.521+0000 7f2b88017700 10 osd.6 12 add_map_bl 12 2216 bytes 2019-12-07T19:33:39.521+0000 7f2b88017700 20 osd.6 12 got_full_map 13, nothing requested 2019-12-07T19:33:39.521+0000 7f2b88017700 20 osd.6 12 handle_osd_map pg_num_history pg_num_history(e13 pg_nums {} deleted_pools ) 2019-12-07T19:33:39.521+0000 7f2b88017700 10 snap_mapper.record_purged_snaps purged_snaps {13={}} 2019-12-07T19:33:39.521+0000 7f2b88017700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:39.521+0000 7f2b88017700 10 osd.6 12 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e13 [1,13] lci=[0,0]) 2019-12-07T19:33:39.521+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7a92000 seq 6 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:cc63972f:::inc_osdmap.13:0# key 0x7f7fffffffffffffffcc6397'/!inc_osdmap.13!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:39.521+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1053 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.521+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~c9c 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0x364 on front/back, now 0x0~1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92000 1 blobs 2019-12-07T19:33:39.521+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:39.521+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1c000~1000/1000,1000,0 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x1c000~1000] 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a28b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a28b60 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28d20 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.blob(0x5557e7a28b60) get_ref 0x0~c9c Blob(0x5557e7a28b60 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28d20 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.blob(0x5557e7a28b60) get_ref init 0x1000, 1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~c9c: 0x0~c9c Blob(0x5557e7a28b60 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0xc9c) SharedBlob(0x5557e7a28d20 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a28d38 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0xc9c 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7508) dirty_range mark inline shard dirty 2019-12-07T19:33:39.521+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c = 0 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:30f0e826:::osdmap.13:0# key 0x7f7fffffffffffffff30f0e8'&!osdmap.13!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:39.521+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1054 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:30f0e826:::osdmap.13:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.521+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:39.521+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~168 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92000 2 blobs 2019-12-07T19:33:39.521+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:39.521+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1d000~2000/1000,2000,0 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x1d000~2000] 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a290a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a290a0 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29260 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.blob(0x5557e7a290a0) get_ref 0x0~1000 Blob(0x5557e7a290a0 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29260 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.blob(0x5557e7a290a0) get_ref init 0x1000, 1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e7a290a0 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7a29260 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a29278 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a29420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a295e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a29420 blob([!~1000,0x1e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a295e0 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29420) get_ref 0x1000~168 Blob(0x5557e7a29420 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e7a295e0 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.blob(0x5557e7a29420) get_ref init 0x2000, 1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x5557e7a29420 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x5557e7a295e0 sbid 0x0)) 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a295f8 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1168 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.521+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7788) dirty_range mark inline shard dirty 2019-12-07T19:33:39.521+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 = 0 2019-12-07T19:33:39.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~12 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92000 0 blobs 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:39.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:39.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:39.522+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92000 0 blobs 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92000 cost 25145 (4 ios * 4000 + 9145 bytes) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92000 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e70c7400,0x5557e70c7680 shared_blobs 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7508) update #-1:cc63972f:::inc_osdmap.13:0# 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:cc63972f:::inc_osdmap.13:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7788) update #-1:30f0e826:::osdmap.13:0# 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7788) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:30f0e826:::osdmap.13:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:39.522+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92000 allocated 0x[1c000~3000] released 0x[] 2019-12-07T19:33:39.522+0000 7f2b88017700 10 freelist allocate 0x1c000~3000 2019-12-07T19:33:39.522+0000 7f2b88017700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92000 prepare 2019-12-07T19:33:39.522+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a92000 2019-12-07T19:33:39.522+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 11 ==== osd_map(12..13 src has 1..13) v4 ==== 4454+0+0 (crc 0 0 0) 0x5557e70c3200 con 0x5557e6f87000 2019-12-07T19:33:39.522+0000 7f2b88017700 20 osd.6 12 OSD::ms_dispatch: osd_map(12..13 src has 1..13) v4 2019-12-07T19:33:39.522+0000 7f2b88017700 10 osd.6 12 do_waiters -- start 2019-12-07T19:33:39.522+0000 7f2b88017700 10 osd.6 12 do_waiters -- finish 2019-12-07T19:33:39.522+0000 7f2b88017700 20 osd.6 12 _dispatch 0x5557e70c3200 osd_map(12..13 src has 1..13) v4 2019-12-07T19:33:39.522+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5557e79b9a00 con 0x5557e6f87000 2019-12-07T19:33:39.522+0000 7f2b88017700 3 osd.6 12 handle_osd_map epochs [12,13], i have 13, src has [1,13] 2019-12-07T19:33:39.522+0000 7f2b88017700 10 osd.6 12 no new maps here, dropping 2019-12-07T19:33:39.522+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92000 aio_wait 2019-12-07T19:33:39.522+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92000 2019-12-07T19:33:39.522+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92000 io_done 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8788 < min_flush_size 524288 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7400 had 1 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7680 had 1 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8811 < min_flush_size 524288 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8811 < min_flush_size 524288 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3dad~226b to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3dad 0x3dad~226b to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x3dad 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xdad 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x18 and padding block with 0xfe8 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x6018 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:39.522+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000235807s (0.000056048s flush + 0.000179759s kv commit) 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92000> 2019-12-07T19:33:39.522+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:39.522+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92000 kv_submitted 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92000 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7a92000 osr 0x5557e76ce500 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 12 _committed_osd_maps 13..13 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 12 advance to epoch 13 (<= last 13 <= newest_map 13) 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e76cf2c0) prepare_write seq 4 0x1b000~1000 crc 2ba63da0 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e76cf2c0) _discard drop 3 0x2000~1000 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e76cf2c0) prepare_write seq 4 0x2000~1000 crc 73855fc0 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 up_epoch is 13 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 boot_epoch is 13 2019-12-07T19:33:39.522+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 osd.6 13 state: booting -> active 2019-12-07T19:33:39.522+0000 7f2b85cfd700 7 osd.6 13 consume_map version 13 2019-12-07T19:33:39.522+0000 7f2b85cfd700 20 osd.6 13 prune_pg_created 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6:0.consume_map 13 (was 12) 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6:1.consume_map 13 (was 12) 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6:2.consume_map 13 (was 12) 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6:3.consume_map 13 (was 12) 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6:4.consume_map 13 (was 12) 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6:5.consume_map 13 (was 12) 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6:6.consume_map 13 (was 12) 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6:7.consume_map 13 (was 12) 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 maybe_update_heartbeat_peers updating 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 adding neighbor peer osd.0 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x5557e7a49000 0x5557e62b6100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x5557e7a49400 0x5557e62b5b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 _add_heartbeat_peer: new peer osd.0 v2:172.21.15.131:6830/37079 v2:172.21.15.131:6826/37079 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 adding neighbor peer osd.5 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x5557e7a49800 0x5557e62b6680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x5557e7a49c00 0x5557e62b6c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 _add_heartbeat_peer: new peer osd.5 v2:172.21.15.109:6806/36936 v2:172.21.15.109:6804/36936 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 adding neighbor peer osd.7 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] conn(0x5557e7af0000 0x5557e62b7180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] conn(0x5557e7af0400 0x5557e62b7700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 _add_heartbeat_peer: new peer osd.7 v2:172.21.15.109:6830/36939 v2:172.21.15.109:6828/36939 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 adding random peer osd.1 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x5557e7af0800 0x5557e747e000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x5557e7af0c00 0x5557e747e580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 _add_heartbeat_peer: new peer osd.1 v2:172.21.15.131:6831/37078 v2:172.21.15.131:6827/37078 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 adding random peer osd.2 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x5557e7af1000 0x5557e747eb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x5557e7af1400 0x5557e747f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 _add_heartbeat_peer: new peer osd.2 v2:172.21.15.131:6808/37077 v2:172.21.15.131:6806/37077 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 adding random peer osd.3 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x5557e7af1800 0x5557e747f600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x5557e7af1c00 0x5557e747fb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 _add_heartbeat_peer: new peer osd.3 v2:172.21.15.131:6816/37076 v2:172.21.15.131:6814/37076 2019-12-07T19:33:39.522+0000 7f2b85cfd700 10 osd.6 13 adding random peer osd.4 2019-12-07T19:33:39.522+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e748e000 0x5557e7480100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.523+0000 7f2b85cfd700 1 --2- >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x5557e748e400 0x5557e7480680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.523+0000 7f2b85cfd700 10 osd.6 13 _add_heartbeat_peer: new peer osd.4 v2:172.21.15.109:6814/36938 v2:172.21.15.109:6812/36938 2019-12-07T19:33:39.523+0000 7f2b85cfd700 10 osd.6 13 maybe_update_heartbeat_peers 7 peers, extras 0,1,2,3,4,5,7 2019-12-07T19:33:39.523+0000 7f2b85cfd700 7 osd.6 13 activate_map version 13 2019-12-07T19:33:39.523+0000 7f2b7ace7700 20 osd.6 13 agent_entry empty queue 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 --2- >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x5557e7a49800 0x5557e62b6680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 --2- >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x5557e7a49800 0x5557e62b6680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.109:6806/36936 says I am v2:172.21.15.109:47024/0 (socket says 172.21.15.109:47024) 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 learned_addr learned my addr 172.21.15.109:0/36937 (peer_addr_for_me v2:172.21.15.109:0/0) 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x5557e7a49000 0x5557e62b6100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 --2- >> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] conn(0x5557e7af0400 0x5557e62b7700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 --2- >> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] conn(0x5557e7af0400 0x5557e62b7700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.109:6828/36939 says I am v2:172.21.15.109:44868/0 (socket says 172.21.15.109:44868) 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 learned_addr learned my addr 172.21.15.109:0/36937 (peer_addr_for_me v2:172.21.15.109:0/0) 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] conn(0x5557e7af0400 0x5557e62b7700 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.533+0000 7f2b95fb3700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x5557e7a49800 0x5557e62b6680 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.533+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e748e000 0x5557e7480100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x5557e7a49c00 0x5557e62b6c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e62f4c00 0x5557e7481180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e748f800 0x5557e7481700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e748fc00 0x5557e74be000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e62f4c00 0x5557e7481180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] conn(0x5557e7af0000 0x5557e62b7180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 12 ==== osd_map(13..13 src has 1..13) v4 ==== 4500+0+0 (crc 0 0 0) 0x5557e70c3d40 con 0x5557e6f87000 2019-12-07T19:33:39.534+0000 7f2b88017700 20 osd.6 13 OSD::ms_dispatch: osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.534+0000 7f2b88017700 10 osd.6 13 do_waiters -- start 2019-12-07T19:33:39.534+0000 7f2b88017700 10 osd.6 13 do_waiters -- finish 2019-12-07T19:33:39.534+0000 7f2b88017700 20 osd.6 13 _dispatch 0x5557e70c3d40 osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc000 0x5557e74be580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e748f800 0x5557e7481700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc400 0x5557e74beb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc800 0x5557e74bf080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b88017700 3 osd.6 13 handle_osd_map epochs [13,13], i have 13, src has [1,13] 2019-12-07T19:33:39.534+0000 7f2b88017700 10 osd.6 13 no new maps here, dropping 2019-12-07T19:33:39.534+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 13 ==== osd_map(13..13 src has 1..13) v4 ==== 4500+0+0 (crc 0 0 0) 0x5557e74ca000 con 0x5557e6f87000 2019-12-07T19:33:39.534+0000 7f2b88017700 20 osd.6 13 OSD::ms_dispatch: osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.534+0000 7f2b88017700 10 osd.6 13 do_waiters -- start 2019-12-07T19:33:39.534+0000 7f2b88017700 10 osd.6 13 do_waiters -- finish 2019-12-07T19:33:39.534+0000 7f2b88017700 20 osd.6 13 _dispatch 0x5557e74ca000 osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.534+0000 7f2b88017700 3 osd.6 13 handle_osd_map epochs [13,13], i have 13, src has [1,13] 2019-12-07T19:33:39.534+0000 7f2b88017700 10 osd.6 13 no new maps here, dropping 2019-12-07T19:33:39.534+0000 7f2b95fb3700 10 osd.6 13 ms_handle_authentication new session 0x5557e7aef380 con 0x5557e62f4c00 entity osd.5 addr 2019-12-07T19:33:39.534+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc800 0x5557e74bf080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b95fb3700 10 osd.6 13 ms_handle_authentication session 0x5557e7aef380 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e748fc00 0x5557e74be000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x5557e7a49c00 0x5557e62b6c00 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc000 0x5557e74be580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x5557e748e400 0x5557e7480680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b957b2700 10 osd.6 13 ms_handle_authentication new session 0x5557e7aef520 con 0x5557e748f800 entity osd.7 addr 2019-12-07T19:33:39.534+0000 7f2b957b2700 10 osd.6 13 ms_handle_authentication session 0x5557e7aef520 osd.7 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e74ccc00 0x5557e74bf600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc400 0x5557e74beb00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e74cd000 0x5557e74bfb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e748e000 0x5557e7480100 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e74cd000 0x5557e74bfb80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cd400 0x5557e74c0100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cd400 0x5557e74c0100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.109:0/36936 conn(0x5557e62f4c00 0x5557e7481180 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f2b95fb3700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x5557e7af1c00 0x5557e747fb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cd800 0x5557e74c0680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.534+0000 7f2b95fb3700 10 osd.6 13 ms_handle_authentication new session 0x5557e7aef860 con 0x5557e74cc400 entity osd.4 addr 2019-12-07T19:33:39.534+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7aefba0 con 0x5557e748fc00 entity osd.4 addr 2019-12-07T19:33:39.534+0000 7f2b95fb3700 10 osd.6 13 ms_handle_authentication session 0x5557e7aef860 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.534+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7aefba0 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.534+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cd800 0x5557e74c0680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.534+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.109:0/36939 conn(0x5557e748f800 0x5557e7481700 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.534+0000 7f2b95fb3700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x5557e7af1000 0x5557e747eb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x5557e7af1800 0x5557e747f600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f2b957b2700 10 osd.6 13 ms_handle_authentication new session 0x5557e7aef6c0 con 0x5557e74cc800 entity osd.7 addr 2019-12-07T19:33:39.535+0000 7f2b957b2700 10 osd.6 13 ms_handle_authentication session 0x5557e7aef6c0 osd.7 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7aefd40 con 0x5557e74cd400 entity osd.2 addr 2019-12-07T19:33:39.535+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7aefd40 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e74ccc00 0x5557e74bf600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x5557e7af1400 0x5557e747f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x5557e748e400 0x5557e7480680 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc000 msgr2=0x5557e74be580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 78 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc000 msgr2=0x5557e74be580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc000 0x5557e74be580 unknown :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cc000 0x5557e74be580 unknown :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.535+0000 7f2b804f2700 20 osd.6 13 heartbeat_reset con 0x5557e74cc000 s 0 2019-12-07T19:33:39.535+0000 7f2b967b4700 0 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e74cd000 msgr2=0x5557e74bfb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:33:39.535+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7516000 con 0x5557e74cd000 entity osd.3 addr 2019-12-07T19:33:39.535+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7516000 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e74cd000 msgr2=0x5557e74bfb80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send send error: (32) Broken pipe 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e74cd000 0x5557e74bfb80 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=1 rx=0 tx=0).write auth done write failed r=-32 ((32) Broken pipe) 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e74cd000 0x5557e74bfb80 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.535+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.109:0/36938 conn(0x5557e74cc400 0x5557e74beb00 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e75161a0 con 0x5557e74cd800 entity osd.3 addr 2019-12-07T19:33:39.535+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e75161a0 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] conn(0x5557e7af0000 0x5557e62b7180 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e748fc00 msgr2=0x5557e74be000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 77 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e748fc00 msgr2=0x5557e74be000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:39.535+0000 7f2b7f4f0700 20 osd.6 13 heartbeat_reset con 0x5557e74cd000 s 0x5557e7516000 2019-12-07T19:33:39.535+0000 7f2b7f4f0700 10 osd.6 13 heartbeat_reset closing (old) failed hb con 0x5557e74cd000 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e748fc00 0x5557e74be000 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e748fc00 0x5557e74be000 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.535+0000 7f2b7f4f0700 20 osd.6 13 heartbeat_reset con 0x5557e748fc00 s 0x5557e7aefba0 2019-12-07T19:33:39.535+0000 7f2b7f4f0700 10 osd.6 13 heartbeat_reset closing (old) failed hb con 0x5557e748fc00 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37077 conn(0x5557e74cd400 0x5557e74c0100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.109:0/36939 conn(0x5557e74cc800 0x5557e74bf080 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37076 conn(0x5557e74cd800 0x5557e74c0680 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f2b957b2700 10 osd.6 13 ms_handle_authentication new session 0x5557e7516000 con 0x5557e74ccc00 entity osd.2 addr 2019-12-07T19:33:39.535+0000 7f2b957b2700 10 osd.6 13 ms_handle_authentication session 0x5557e7516000 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.535+0000 7f2b95fb3700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x5557e7af1c00 0x5557e747fb80 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.535+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cdc00 0x5557e74c0c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.536+0000 7f2b95fb3700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x5557e7af1000 0x5557e747eb00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x5557e7af1800 0x5557e747f600 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e74cdc00 0x5557e74c0c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37077 conn(0x5557e74ccc00 0x5557e74bf600 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x5557e7af1400 0x5557e747f080 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e7518000 0x5557e74c1180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.536+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7516340 con 0x5557e74cdc00 entity osd.1 addr 2019-12-07T19:33:39.536+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7516340 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e7518400 0x5557e74c1700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e7518000 0x5557e74c1180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37078 conn(0x5557e74cdc00 0x5557e74c0c00 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e7518400 0x5557e74c1700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x5557e7af0c00 0x5557e747e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7516680 con 0x5557e7518400 entity osd.1 addr 2019-12-07T19:33:39.536+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7516680 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7516820 con 0x5557e7518000 entity osd.0 addr 2019-12-07T19:33:39.536+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7516820 osd.0 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e7518800 0x5557e7526000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.537+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x5557e7a49400 0x5557e62b5b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.537+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37078 conn(0x5557e7518400 0x5557e74c1700 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x5557e7af0800 0x5557e747e000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.537+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e7518800 0x5557e7526000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.537+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37079 conn(0x5557e7518000 0x5557e74c1180 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x5557e7af0c00 0x5557e747e580 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f2b95fb3700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] conn(0x5557e7a49000 0x5557e62b6100 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e75169c0 con 0x5557e7518800 entity osd.0 addr 2019-12-07T19:33:39.537+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e75169c0 osd.0 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.537+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37079 conn(0x5557e7518800 0x5557e7526000 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] conn(0x5557e7a49400 0x5557e62b5b80 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x5557e7af0800 0x5557e747e000 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.552+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:33:39.552+0000 7f2b8ad07700 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock osr 0x5557e76ce500 2 ios pending 2019-12-07T19:33:39.552+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock seq 4 0x2000~1000 2019-12-07T19:33:39.552+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock write 0x2000~1000 crc 73855fc0 2019-12-07T19:33:39.552+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock seq 4 0x1b000~1000 2019-12-07T19:33:39.552+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock write 0x1b000~1000 crc 2ba63da0 2019-12-07T19:33:39.552+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.552+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_aio_finish osr 0x5557e76ce500 2019-12-07T19:33:39.552+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_aio_finish dequeueing 2019-12-07T19:33:39.558+0000 7f2b92fad700 10 osd.6 13 tick 2019-12-07T19:33:39.558+0000 7f2b92fad700 10 osd.6 13 do_waiters -- start 2019-12-07T19:33:39.558+0000 7f2b92fad700 10 osd.6 13 do_waiters -- finish 2019-12-07T19:33:39.558+0000 7f2b92fad700 20 osd.6 13 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:39.593+0000 7f2b9171a700 10 osd.6 13 tick_without_osd_lock 2019-12-07T19:33:39.593+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffe1000/0x40000000/0x1680000000, data 0x724e/0x1d000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:39.593+0000 7f2b9171a700 25 osd.6 13 heartbeat_check we haven't sent ping to osd.0 yet, skipping 2019-12-07T19:33:39.593+0000 7f2b9171a700 25 osd.6 13 heartbeat_check we haven't sent ping to osd.1 yet, skipping 2019-12-07T19:33:39.593+0000 7f2b9171a700 25 osd.6 13 heartbeat_check we haven't sent ping to osd.2 yet, skipping 2019-12-07T19:33:39.593+0000 7f2b9171a700 25 osd.6 13 heartbeat_check we haven't sent ping to osd.3 yet, skipping 2019-12-07T19:33:39.593+0000 7f2b9171a700 25 osd.6 13 heartbeat_check we haven't sent ping to osd.4 yet, skipping 2019-12-07T19:33:39.593+0000 7f2b9171a700 25 osd.6 13 heartbeat_check we haven't sent ping to osd.5 yet, skipping 2019-12-07T19:33:39.593+0000 7f2b9171a700 25 osd.6 13 heartbeat_check we haven't sent ping to osd.7 yet, skipping 2019-12-07T19:33:39.593+0000 7f2b9171a700 20 osd.6 13 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:39.593+0000 7f2b9171a700 20 osd.6 13 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:39.593+0000 7f2b9171a700 20 osd.6 13 scrub_load_below_threshold loadavg per cpu 0.0725 < max 5 = yes 2019-12-07T19:33:39.593+0000 7f2b9171a700 20 osd.6 13 sched_scrub load_is_low=1 2019-12-07T19:33:39.593+0000 7f2b9171a700 20 osd.6 13 sched_scrub done 2019-12-07T19:33:39.593+0000 7f2b9171a700 10 osd.6 13 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:39.593+0000 7f2b9171a700 20 osd.6 13 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:39.593+0000 7f2b9171a700 10 osd.6 13 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:39.593+0000 7f2b9171a700 20 osd.6 13 send_beacon sending 2019-12-07T19:33:39.593+0000 7f2b9171a700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_beacon(pgs [] lec 12 last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 v13) v2 -- 0x5557e79b8e00 con 0x5557e6f87000 2019-12-07T19:33:39.602+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.653+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.703+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.736+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e7518c00 0x5557e7526580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.736+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e7519000 0x5557e7526b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.736+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e7518c00 0x5557e7526580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.736+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e7519000 0x5557e7526b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.736+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7516b60 con 0x5557e7518c00 entity osd.5 addr 2019-12-07T19:33:39.736+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7516b60 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.736+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7516d00 con 0x5557e7519000 entity osd.4 addr 2019-12-07T19:33:39.736+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7516d00 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.736+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e7519400 0x5557e7527080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.736+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e7519400 0x5557e7527080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.736+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.109:0/36936 conn(0x5557e7518c00 0x5557e7526580 crc :-1 s=READY pgs=9 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.736+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.109:0/36938 conn(0x5557e7519000 0x5557e7526b00 crc :-1 s=READY pgs=9 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.736+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication new session 0x5557e7516ea0 con 0x5557e7519400 entity osd.3 addr 2019-12-07T19:33:39.736+0000 7f2b967b4700 10 osd.6 13 ms_handle_authentication session 0x5557e7516ea0 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.737+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37076 conn(0x5557e7519400 0x5557e7527080 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.753+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.803+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.853+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.903+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.953+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.003+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.054+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.104+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.154+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.204+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.254+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.304+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.354+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.405+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.455+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.505+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.519+0000 7f2b83cf9700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:40.519+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.6) v3 -- 0x5557e6e8f600 con 0x5557e7a48000 2019-12-07T19:33:40.519+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:40.519+0000 7f2b88017700 10 osd.6 13 ms_handle_connect con 0x5557e7a48000 2019-12-07T19:33:40.519+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x5557e7a48000 0x5557e62b5600 crc :-1 s=READY pgs=36 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:40.520+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mgr.4099 v2:172.21.15.131:6800/36137 1 ==== mgrconfigure(period=5, threshold=5) v3 ==== 12+0+0 (crc 0 0 0) 0x5557e74f0600 con 0x5557e7a48000 2019-12-07T19:33:40.520+0000 7f2b88017700 10 osd.6 13 setting 0 queries 2019-12-07T19:33:40.520+0000 7f2b88017700 20 osd.6 13 reports for 0 queries 2019-12-07T19:33:40.520+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.6 +109-0 packed 1198 daemon_metrics=2) v8 -- 0x5557e79c4380 con 0x5557e7a48000 2019-12-07T19:33:40.520+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x5557e79c4700 con 0x5557e7a48000 2019-12-07T19:33:40.527+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 14 ==== osd_map(13..14 src has 1..14) v4 ==== 3816+0+0 (crc 0 0 0) 0x5557e74ca6c0 con 0x5557e6f87000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 osd.6 13 OSD::ms_dispatch: osd_map(13..14 src has 1..14) v4 2019-12-07T19:33:40.527+0000 7f2b88017700 10 osd.6 13 do_waiters -- start 2019-12-07T19:33:40.527+0000 7f2b88017700 10 osd.6 13 do_waiters -- finish 2019-12-07T19:33:40.527+0000 7f2b88017700 20 osd.6 13 _dispatch 0x5557e74ca6c0 osd_map(13..14 src has 1..14) v4 2019-12-07T19:33:40.527+0000 7f2b88017700 3 osd.6 13 handle_osd_map epochs [13,14], i have 13, src has [1,14] 2019-12-07T19:33:40.527+0000 7f2b88017700 10 osd.6 13 handle_osd_map got inc map for epoch 14 2019-12-07T19:33:40.527+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:30f0e826:::osdmap.13:0# 0x0~0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7a290a0 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7a29260 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7a29420 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x5557e7a295e0 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 = 4456 2019-12-07T19:33:40.527+0000 7f2b88017700 10 osd.6 13 add_map_bl 13 4456 bytes 2019-12-07T19:33:40.527+0000 7f2b88017700 20 osd.6 13 got_full_map 14, nothing requested 2019-12-07T19:33:40.527+0000 7f2b88017700 20 osd.6 13 handle_osd_map pg_num_history pg_num_history(e14 pg_nums {} deleted_pools ) 2019-12-07T19:33:40.527+0000 7f2b88017700 10 snap_mapper.record_purged_snaps purged_snaps {14={}} 2019-12-07T19:33:40.527+0000 7f2b88017700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:40.527+0000 7f2b88017700 10 osd.6 13 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e14 [1,14] lci=[13,14]) 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7a92300 seq 7 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c3e3972f:::inc_osdmap.14:0# key 0x7f7fffffffffffffffc3e397'/!inc_osdmap.14!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:40.527+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1055 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~218 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xde8 on front/back, now 0x0~1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92300 1 blobs 2019-12-07T19:33:40.527+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:40.527+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1f000~1000/1000,1000,0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x1f000~1000] 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a28620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a287e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a28620 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a287e0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e7a28620) get_ref 0x0~218 Blob(0x5557e7a28620 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a287e0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e7a28620) get_ref init 0x1000, 1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~218: 0x0~218 Blob(0x5557e7a28620 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x218) SharedBlob(0x5557e7a287e0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a287f8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x218 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7a08) dirty_range mark inline shard dirty 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 = 0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3b30e826:::osdmap.14:0# key 0x7f7fffffffffffffff3b30e8'&!osdmap.14!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:40.527+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1056 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3b30e826:::osdmap.14:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~168 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92300 2 blobs 2019-12-07T19:33:40.527+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:40.527+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x20000~2000/1000,2000,0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x20000~2000] 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e6f54460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e6f54460 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91f10 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e6f54460) get_ref 0x0~1000 Blob(0x5557e6f54460 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a91f10 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e6f54460) get_ref init 0x1000, 1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e6f54460 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7a91f10 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91f28 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e6f54150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d90a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e6f54150 blob([!~1000,0x21000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d90a0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e6f54150) get_ref 0x1000~168 Blob(0x5557e6f54150 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e74d90a0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e6f54150) get_ref init 0x2000, 1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x5557e6f54150 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x5557e74d90a0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e74d90b8 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1168 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7c88) dirty_range mark inline shard dirty 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 = 0 2019-12-07T19:33:40.527+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~12 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92300 0 blobs 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:40.527+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:40.527+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:40.527+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92300 0 blobs 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92300 cost 22453 (4 ios * 4000 + 6453 bytes) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92300 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e70c7900,0x5557e70c7b80 shared_blobs 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7a08) update #-1:c3e3972f:::inc_osdmap.14:0# 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c3e3972f:::inc_osdmap.14:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7c88) update #-1:3b30e826:::osdmap.14:0# 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7c88) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3b30e826:::osdmap.14:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:40.527+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92300 allocated 0x[1f000~3000] released 0x[] 2019-12-07T19:33:40.527+0000 7f2b88017700 10 freelist allocate 0x1f000~3000 2019-12-07T19:33:40.527+0000 7f2b88017700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 prepare 2019-12-07T19:33:40.527+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a92300 2019-12-07T19:33:40.527+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 aio_wait 2019-12-07T19:33:40.527+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92300 2019-12-07T19:33:40.527+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 io_done 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8168 < min_flush_size 524288 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8799 < min_flush_size 524288 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7900 had 1 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7b80 had 1 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8870 < min_flush_size 524288 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8870 < min_flush_size 524288 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x6018~22a6 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x6018 0x6018~22a6 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x6018 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x18 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x2be and padding block with 0xd42 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x82be 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:40.527+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 1 in 0.000231656s (0.000051043s flush + 0.000180613s kv commit) 2019-12-07T19:33:40.527+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:40.527+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:40.527+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92300> 2019-12-07T19:33:40.527+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <0x5557e76cf2c0> 2019-12-07T19:33:40.527+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 kv_submitted 2019-12-07T19:33:40.527+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92300 2019-12-07T19:33:40.527+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7a92300 osr 0x5557e76ce500 2019-12-07T19:33:40.527+0000 7f2b85cfd700 10 osd.6 13 _committed_osd_maps 14..14 2019-12-07T19:33:40.527+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79edb80) prepare_write seq 5 0x1b000~1000 crc 4c6c8a7 2019-12-07T19:33:40.527+0000 7f2b85cfd700 10 osd.6 13 advance to epoch 14 (<= last 14 <= newest_map 14) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79edb80) prepare_write seq 5 0x2000~1000 crc 1f87da88 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 deferred_cleanup 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121800 onodes 0x5557e62a1b80 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121800 done 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 deferred_cleanup 2019-12-07T19:33:40.528+0000 7f2b85cfd700 7 osd.6 14 consume_map version 14 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121800 [] 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:40.528+0000 7f2b85cfd700 20 osd.6 14 prune_pg_created 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6:0.consume_map 14 (was 13) 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 deferred_cleanup 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6:1.consume_map 14 (was 13) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 0x5557e62a1b80 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6:2.consume_map 14 (was 13) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6:3.consume_map 14 (was 13) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 deferred_cleanup 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6:4.consume_map 14 (was 13) 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6:5.consume_map 14 (was 13) 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6:6.consume_map 14 (was 13) 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6:7.consume_map 14 (was 13) 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 deferred_cleanup 2019-12-07T19:33:40.528+0000 7f2b85cfd700 10 osd.6 14 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:40.528+0000 7f2b85cfd700 7 osd.6 14 activate_map version 14 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121500 onodes 0x5557e62a1b80,0x5557e70c6000,0x5557e70c6280,0x5557e70c6500,0x5557e70c6780,0x5557e70c6a00,0x5557e70c6c80,0x5557e70c6f00,0x5557e70c7180,0x5557e70ce500,0x5557e70cf180,0x5557e70cf400,0x5557e70cf680,0x5557e70cf900,0x5557e70cfb80,0x5557e79ee000,0x5557e79ee280,0x5557e79ee500,0x5557e79ee780,0x5557e79eea00,0x5557e79eec80,0x5557e79eef00,0x5557e79ef180,0x5557e79ef400,0x5557e79ef680,0x5557e79ef900,0x5557e79efb80 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a29d68 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa630 space 0x5557e7a29d68 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a29eb8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa3f0 space 0x5557e7a29eb8 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a29f28 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa6c0 space 0x5557e7a29f28 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90088 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa360 space 0x5557e7a90088 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90248 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa5a0 space 0x5557e7a90248 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90398 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa510 space 0x5557e7a90398 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a904e8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa480 space 0x5557e7a904e8 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90718 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa750 space 0x5557e7a90718 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90868 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa7e0 space 0x5557e7a90868 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a909b8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa870 space 0x5557e7a909b8 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90b08 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa900 space 0x5557e7a90b08 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90c58 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa990 space 0x5557e7a90c58 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90d38 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aaa20 space 0x5557e7a90d38 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90e88 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aaab0 space 0x5557e7a90e88 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a90fd8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aab40 space 0x5557e7a90fd8 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a91128 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e7a91128 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a91278 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aac60 space 0x5557e7a91278 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a913c8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aacf0 space 0x5557e7a913c8 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a91518 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aad80 space 0x5557e7a91518 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a91668 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aae10 space 0x5557e7a91668 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a917b8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aaea0 space 0x5557e7a917b8 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a91908 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aaf30 space 0x5557e7a91908 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a91a58 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aafc0 space 0x5557e7a91a58 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a91ba8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab050 space 0x5557e7a91ba8 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 done 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92000 deferred_cleanup 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121500 [] 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92000 deferred_cleanup 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92000 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e70c7400,0x5557e70c7680 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a28d38 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab200 space 0x5557e7a28d38 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a29278 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab290 space 0x5557e7a29278 0x0~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a295f8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab320 space 0x5557e7a295f8 0x1000~1000 writing nocache) 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92000 done 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92300 deferred_queued 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92000 [] 2019-12-07T19:33:40.528+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:40.528+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:40.528+0000 7f2b7ace7700 20 osd.6 14 agent_entry empty queue 2019-12-07T19:33:40.544+0000 7f2b92fad700 10 osd.6 14 tick 2019-12-07T19:33:40.544+0000 7f2b92fad700 10 osd.6 14 do_waiters -- start 2019-12-07T19:33:40.544+0000 7f2b92fad700 10 osd.6 14 do_waiters -- finish 2019-12-07T19:33:40.544+0000 7f2b92fad700 20 osd.6 14 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:40.555+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.605+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.622+0000 7f2b9171a700 10 osd.6 14 tick_without_osd_lock 2019-12-07T19:33:40.622+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffde000/0x40000000/0x1680000000, data 0x85ce/0x20000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:40.622+0000 7f2b9171a700 25 osd.6 14 heartbeat_check we haven't sent ping to osd.0 yet, skipping 2019-12-07T19:33:40.622+0000 7f2b9171a700 25 osd.6 14 heartbeat_check we haven't sent ping to osd.1 yet, skipping 2019-12-07T19:33:40.622+0000 7f2b9171a700 25 osd.6 14 heartbeat_check we haven't sent ping to osd.2 yet, skipping 2019-12-07T19:33:40.622+0000 7f2b9171a700 25 osd.6 14 heartbeat_check we haven't sent ping to osd.3 yet, skipping 2019-12-07T19:33:40.622+0000 7f2b9171a700 25 osd.6 14 heartbeat_check we haven't sent ping to osd.4 yet, skipping 2019-12-07T19:33:40.622+0000 7f2b9171a700 25 osd.6 14 heartbeat_check we haven't sent ping to osd.5 yet, skipping 2019-12-07T19:33:40.622+0000 7f2b9171a700 25 osd.6 14 heartbeat_check we haven't sent ping to osd.7 yet, skipping 2019-12-07T19:33:40.622+0000 7f2b9171a700 20 osd.6 14 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:40.622+0000 7f2b9171a700 10 osd.6 14 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:40.622+0000 7f2b9171a700 20 osd.6 14 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:40.622+0000 7f2b9171a700 10 osd.6 14 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:40.655+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.705+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.755+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.806+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.856+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.906+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.956+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.006+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.056+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.106+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.156+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.207+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.236+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.2 172.21.15.131:0/37077 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.707221878s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f0400 con 0x5557e74cd400 2019-12-07T19:33:41.236+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.2 172.21.15.131:0/37077 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.707221878s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f1400 con 0x5557e74ccc00 2019-12-07T19:33:41.236+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016453857s,]) 2019-12-07T19:33:41.236+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.690768021s delta_ub 18446744073.693097759s) v5 -- 0x5557e74f0a00 con 0x5557e74cd400 2019-12-07T19:33:41.236+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.236+0000 7f2b957b2700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016385870s,]) 2019-12-07T19:33:41.236+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.690836008s delta_ub 18446744073.693165746s) v5 -- 0x5557e74f1600 con 0x5557e74ccc00 2019-12-07T19:33:41.236+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.236+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e7519c00 0x5557e7527b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.236+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e7519c00 0x5557e7527b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.237+0000 7f2b957b2700 10 osd.6 14 ms_handle_authentication new session 0x5557e75164e0 con 0x5557e7519c00 entity osd.2 addr 2019-12-07T19:33:41.237+0000 7f2b957b2700 10 osd.6 14 ms_handle_authentication session 0x5557e75164e0 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.237+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e7519c00 msgr2=0x5557e7527b80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 91 2019-12-07T19:33:41.237+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e7519c00 msgr2=0x5557e7527b80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.237+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e7519c00 0x5557e7527b80 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.237+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e7519c00 0x5557e7527b80 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.237+0000 7f2b834f8700 2 osd.6 14 ms_handle_reset con 0x5557e7519c00 session 0x5557e75164e0 2019-12-07T19:33:41.237+0000 7f2b95fb3700 10 osd.6 14 new session (outgoing) 0x5557e7aefa00 con=0x5557e7519800 addr=v2:172.21.15.131:6804/37077 2019-12-07T19:33:41.237+0000 7f2b834f8700 10 osd.6 14 ms_handle_connect con 0x5557e7519800 2019-12-07T19:33:41.237+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 crc :-1 s=READY pgs=10 cs=0 l=0 rx=0 tx=0).ready entity=osd.2 client_cookie=18704efb08e1a046 server_cookie=2a616824987376d2 in_seq=0 out_seq=0 2019-12-07T19:33:41.240+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.5 172.21.15.109:0/36936 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.706869389s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e62b8c00 con 0x5557e62f4c00 2019-12-07T19:33:41.240+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011544466s,]) 2019-12-07T19:33:41.240+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.695324923s delta_ub 18446744073.698007150s) v5 -- 0x5557e74f1c00 con 0x5557e62f4c00 2019-12-07T19:33:41.240+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.240+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.5 172.21.15.109:0/36936 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.706869389s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7518c00 2019-12-07T19:33:41.240+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011447985s,]) 2019-12-07T19:33:41.240+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.695421404s delta_ub 18446744073.698103631s) v5 -- 0x5557e74f0a00 con 0x5557e7518c00 2019-12-07T19:33:41.241+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.241+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79ca400 0x5557e7528680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.241+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79ca400 0x5557e7528680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.241+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 crc :-1 s=START_CONNECT pgs=0 cs=0 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:41.241+0000 7f2b957b2700 10 osd.6 14 ms_handle_authentication new session 0x5557e7aefba0 con 0x5557e79ca400 entity osd.5 addr 2019-12-07T19:33:41.241+0000 7f2b957b2700 10 osd.6 14 ms_handle_authentication session 0x5557e7aefba0 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.241+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca400 0x5557e7528680 crc :-1 s=SESSION_ACCEPTING pgs=5 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x5557e79ca000 socket by this connection's socket 2019-12-07T19:33:41.242+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca400 0x5557e7528680 crc :-1 s=SESSION_ACCEPTING pgs=5 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.242+0000 7f2b834f8700 2 osd.6 14 ms_handle_reset con 0x5557e79ca400 session 0x5557e7aefba0 2019-12-07T19:33:41.242+0000 7f2b957b2700 10 osd.6 14 new session (incoming)0x5557e7517520 con=0x5557e79ca000 addr=v2:172.21.15.109:6802/36936 must have raced with connect 2019-12-07T19:33:41.242+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=READY pgs=5 cs=0 l=0 rx=0 tx=0).ready entity=osd.5 client_cookie=fce9122ddfb0a0db server_cookie=6547c7d58203b459 in_seq=0 out_seq=0 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.707537742s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f0400 con 0x5557e7519400 2019-12-07T19:33:41.250+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002629366s,]) 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.704908376s delta_ub 18446744073.706922250s) v5 -- 0x5557e74f0a00 con 0x5557e7519400 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x5557e79ca800 0x5557e7528c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.707537742s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e74cd800 2019-12-07T19:33:41.250+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002378551s,]) 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.705159191s delta_ub 18446744073.707173065s) v5 -- 0x5557e74f0a00 con 0x5557e74cd800 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.1 172.21.15.131:0/37078 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.706948734s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e74cdc00 2019-12-07T19:33:41.250+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001753065s,]) 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.705195669s delta_ub 18446744073.707798551s) v5 -- 0x5557e74f0c00 con 0x5557e74cdc00 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.1 172.21.15.131:0/37078 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.706948734s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f0400 con 0x5557e7518400 2019-12-07T19:33:41.250+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001672514s,]) 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.705276220s delta_ub 18446744073.707879102s) v5 -- 0x5557e74f0c00 con 0x5557e7518400 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb000 0x5557e7529700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.250+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x5557e79ca800 0x5557e7528c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.250+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb000 0x5557e7529700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.250+0000 7f2b967b4700 10 osd.6 14 ms_handle_authentication new session 0x5557e7517040 con 0x5557e79cb000 entity osd.3 addr 2019-12-07T19:33:41.250+0000 7f2b967b4700 10 osd.6 14 ms_handle_authentication session 0x5557e7517040 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.251+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x5557e79cb000 0x5557e7529700 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, this connection loses to existing=0x5557e79ca800 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x5557e79cb000 msgr2=0x5557e7529700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 94 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x5557e79cb000 msgr2=0x5557e7529700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x5557e79cb000 0x5557e7529700 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x5557e79cb000 0x5557e7529700 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.251+0000 7f2b834f8700 2 osd.6 14 ms_handle_reset con 0x5557e79cb000 session 0x5557e7517040 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb400 0x5557e7a96000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb400 0x5557e7a96000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb400 msgr2=0x5557e7a96000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 94 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb400 msgr2=0x5557e7a96000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb400 0x5557e7a96000 unknown :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.251+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb400 0x5557e7a96000 unknown :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.251+0000 7f2b834f8700 2 osd.6 14 ms_handle_reset con 0x5557e79cb400 session 0 2019-12-07T19:33:41.251+0000 7f2b957b2700 10 osd.6 14 new session (outgoing) 0x5557e75176c0 con=0x5557e79cac00 addr=v2:172.21.15.131:6823/37078 2019-12-07T19:33:41.251+0000 7f2b834f8700 10 osd.6 14 ms_handle_connect con 0x5557e79cac00 2019-12-07T19:33:41.251+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 crc :-1 s=READY pgs=10 cs=0 l=0 rx=0 tx=0).ready entity=osd.1 client_cookie=949c5c84788cefc8 server_cookie=3b65d3eec6e35050 in_seq=0 out_seq=0 2019-12-07T19:33:41.251+0000 7f2b957b2700 10 osd.6 14 new session (outgoing) 0x5557e7517860 con=0x5557e79ca800 addr=v2:172.21.15.131:6812/37076 2019-12-07T19:33:41.251+0000 7f2b834f8700 10 osd.6 14 ms_handle_connect con 0x5557e79ca800 2019-12-07T19:33:41.252+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x5557e79ca800 0x5557e7528c00 crc :-1 s=READY pgs=12 cs=0 l=0 rx=0 tx=0).ready entity=osd.3 client_cookie=1c3854e87bbbe34e server_cookie=a7db2ebb2e47ec6c in_seq=0 out_seq=0 2019-12-07T19:33:41.252+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.0 172.21.15.131:0/37079 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7518000 2019-12-07T19:33:41.252+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709528770s,]) 2019-12-07T19:33:41.252+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706775745s delta_ub 0.000022846s) v5 -- 0x5557e74f0400 con 0x5557e7518000 2019-12-07T19:33:41.252+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 0x5557e7a96580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.252+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.0 172.21.15.131:0/37079 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 ==== 2025+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7518800 2019-12-07T19:33:41.252+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709451791s,]) 2019-12-07T19:33:41.252+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706852724s delta_ub 0.000099825s) v5 -- 0x5557e74f0a00 con 0x5557e7518800 2019-12-07T19:33:41.252+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 0x5557e7a96580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cbc00 0x5557e7a96b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.252+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cbc00 0x5557e7a96b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cbc00 msgr2=0x5557e7a96b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 95 2019-12-07T19:33:41.252+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cbc00 msgr2=0x5557e7a96b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.252+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cbc00 0x5557e7a96b00 unknown :-1 s=HELLO_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.252+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cbc00 0x5557e7a96b00 unknown :-1 s=HELLO_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.252+0000 7f2b834f8700 2 osd.6 14 ms_handle_reset con 0x5557e79cbc00 session 0 2019-12-07T19:33:41.252+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] reap_dead start 2019-12-07T19:33:41.252+0000 7f2b967b4700 10 osd.6 14 new session (outgoing) 0x5557e75171e0 con=0x5557e79cb800 addr=v2:172.21.15.131:6822/37079 2019-12-07T19:33:41.252+0000 7f2b834f8700 10 osd.6 14 ms_handle_connect con 0x5557e79cb800 2019-12-07T19:33:41.252+0000 7f2b724d6700 5 osd.6 14 heartbeat osd_stat(store_statfs(0x163ffde000/0x40000000/0x1680000000, data 0x85ce/0x20000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,7] op hist []) 2019-12-07T19:33:41.252+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 0x5557e7a96580 crc :-1 s=READY pgs=15 cs=0 l=0 rx=0 tx=0).ready entity=osd.0 client_cookie=66df44187c63ea59 server_cookie=2175ee4a8507a6a8 in_seq=0 out_seq=0 2019-12-07T19:33:41.252+0000 7f2b724d6700 20 osd.6 14 check_full_status cur ratio 1.44111e-06, physical ratio 1.44111e-06, new state none 2019-12-07T19:33:41.252+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 0.000099825s) v5 -- 0x5557e7a9a000 con 0x5557e7a49000 2019-12-07T19:33:41.252+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 0.000099825s) v5 -- 0x5557e7a9a200 con 0x5557e7a49400 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 18446744073.707879102s) v5 -- 0x5557e7a9a400 con 0x5557e7af0800 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 18446744073.707879102s) v5 -- 0x5557e7a9a600 con 0x5557e7af0c00 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 18446744073.693165746s) v5 -- 0x5557e7a9a800 con 0x5557e7af1000 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 18446744073.693165746s) v5 -- 0x5557e7a9aa00 con 0x5557e7af1400 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 18446744073.707173065s) v5 -- 0x5557e7a9a200 con 0x5557e7af1800 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 18446744073.707173065s) v5 -- 0x5557e74f1800 con 0x5557e7af1c00 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s) v5 -- 0x5557e74f1600 con 0x5557e748e000 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s) v5 -- 0x5557e74f1400 con 0x5557e748e400 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 18446744073.698103631s) v5 -- 0x5557e7a9b400 con 0x5557e7a49800 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 18446744073.698103631s) v5 -- 0x5557e7a9b600 con 0x5557e7a49c00 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s) v5 -- 0x5557e7a9b800 con 0x5557e7af0000 2019-12-07T19:33:41.253+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s) v5 -- 0x5557e7a9ba00 con 0x5557e7af0400 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6831/37078 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.710323675s delta_ub 0.002658122s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0000 con 0x5557e7af0800 2019-12-07T19:33:41.253+0000 7f2b957b2700 25 osd.6 14 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253456+0000 last_rx_front 0.000000 2019-12-07T19:33:41.253+0000 7f2b957b2700 10 osd.6 14 maybe_share_map con 0x5557e79cac00 v2:172.21.15.131:6823/37078 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.253+0000 7f2b957b2700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002362071s,0.002658122s]) 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6806/37077 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.725161637s delta_ub 0.017496084s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0000 con 0x5557e7af1400 2019-12-07T19:33:41.253+0000 7f2b957b2700 25 osd.6 14 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253500+0000 2019-12-07T19:33:41.253+0000 7f2b957b2700 10 osd.6 14 maybe_share_map con 0x5557e7519800 v2:172.21.15.131:6804/37077 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.253+0000 7f2b957b2700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017155723s,0.017496084s]) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6830/37079 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.708049322s delta_ub 0.000383769s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e62b8c00 con 0x5557e7a49000 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6814/36938 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.702648633s delta_ub 18446744073.704534696s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0000 con 0x5557e748e000 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6827/37078 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.710437748s delta_ub 0.002772195s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a600 con 0x5557e7af0c00 2019-12-07T19:33:41.253+0000 7f2b95fb3700 25 osd.6 14 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253538+0000 last_rx_front 0.000000 2019-12-07T19:33:41.253+0000 7f2b95fb3700 10 osd.6 14 maybe_share_map con 0x5557e79cb800 v2:172.21.15.131:6822/37079 map epoch 0 -> 13 (as per caller) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 10 osd.6 14 send_incremental_map 13 -> 14 to 0x5557e79cb800 v2:172.21.15.131:6822/37079 2019-12-07T19:33:41.253+0000 7f2b95fb3700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~0 2019-12-07T19:33:41.253+0000 7f2b95fb3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~218 size 0x218 (536) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:41.253+0000 7f2b95fb3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7a28620 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x218) SharedBlob(0x5557e7a287e0 sbid 0x0)) need 0x0~218 cache has 0x[0~218] 2019-12-07T19:33:41.253+0000 7f2b95fb3700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 = 536 2019-12-07T19:33:41.253+0000 7f2b95fb3700 10 osd.6 14 add_map_inc_bl 14 536 bytes 2019-12-07T19:33:41.253+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(14..14 src has 1..14) v4 -- 0x5557e6289200 con 0x5557e79cb800 2019-12-07T19:33:41.253+0000 7f2b95fb3700 10 osd.6 14 maybe_share_map con 0x5557e79cb800 v2:172.21.15.131:6822/37079 map epoch 13 -> 14 (shared) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.000005548s,0.000383769s]) 2019-12-07T19:33:41.253+0000 7f2b957b2700 25 osd.6 14 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253617+0000 last_rx_front 0.000000 2019-12-07T19:33:41.253+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6808/37077 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.725133115s delta_ub 0.017467562s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e62b8c00 con 0x5557e7af1000 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 0x5557e7529700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.253+0000 7f2b957b2700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 0 peer_clock_delta [18446744073.704077508s,18446744073.704534696s]) 2019-12-07T19:33:41.253+0000 7f2b967b4700 25 osd.6 14 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253456+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253658+0000 2019-12-07T19:33:41.253+0000 7f2b967b4700 25 osd.6 14 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:41.253160+0000) and older pending ping(s) 2019-12-07T19:33:41.253+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002274181s,0.002772195s]) 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6826/37079 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.708155789s delta_ub 0.000490236s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0000 con 0x5557e7a49400 2019-12-07T19:33:41.253+0000 7f2b957b2700 25 osd.6 14 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253538+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253681+0000 2019-12-07T19:33:41.253+0000 7f2b957b2700 25 osd.6 14 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:41.253160+0000) and older pending ping(s) 2019-12-07T19:33:41.253+0000 7f2b957b2700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709520297s,0.000490236s]) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 25 osd.6 14 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253703+0000 last_rx_front 2019-12-07T19:33:41.253500+0000 2019-12-07T19:33:41.253+0000 7f2b95fb3700 25 osd.6 14 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:41.253160+0000) and older pending ping(s) 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb400 0x5557e7a97080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.253+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016924345s,0.017467562s]) 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb000 0x5557e7a97600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6830/36939 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.688961995s delta_ub 18446744073.690848058s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7af0000 2019-12-07T19:33:41.253+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6806/36936 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.719768757s delta_ub 0.012103204s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e62b8c00 con 0x5557e7a49800 2019-12-07T19:33:41.253+0000 7f2b967b4700 25 osd.6 14 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253773+0000 last_rx_front 0.000000 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6804/36936 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.719820385s delta_ub 0.012154832s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b200 con 0x5557e7a49c00 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x5557e79f4000 0x5557e7a97b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.253+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.7 up_from 0 peer_clock_delta [18446744073.690234381s,18446744073.690848058s]) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 25 osd.6 14 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253810+0000 last_rx_front 0.000000 2019-12-07T19:33:41.253+0000 7f2b95fb3700 10 osd.6 14 maybe_share_map con 0x5557e79ca000 v2:172.21.15.109:6802/36936 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011452428s,0.012103204s]) 2019-12-07T19:33:41.253+0000 7f2b957b2700 25 osd.6 14 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253810+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253828+0000 2019-12-07T19:33:41.253+0000 7f2b957b2700 25 osd.6 14 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:41.253160+0000) and older pending ping(s) 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6812/36938 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.702840248s delta_ub 18446744073.704726311s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e748e400 2019-12-07T19:33:41.253+0000 7f2b957b2700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011486269s,0.012154832s]) 2019-12-07T19:33:41.253+0000 7f2b967b4700 25 osd.6 14 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253617+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253852+0000 2019-12-07T19:33:41.253+0000 7f2b967b4700 25 osd.6 14 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:41.253160+0000) and older pending ping(s) 2019-12-07T19:33:41.253+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 0 peer_clock_delta [18446744073.704034436s,18446744073.704726311s]) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6814/37076 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.711034837s delta_ub 0.003369284s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e7af1c00 2019-12-07T19:33:41.253+0000 7f2b95fb3700 25 osd.6 14 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253876+0000 2019-12-07T19:33:41.253+0000 7f2b95fb3700 10 osd.6 14 maybe_share_map con 0x5557e79ca800 v2:172.21.15.131:6812/37076 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002652520s,0.003369284s]) 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb400 0x5557e7a97080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6816/37076 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.711178884s delta_ub 0.003513331s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7af1800 2019-12-07T19:33:41.253+0000 7f2b967b4700 25 osd.6 14 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253895+0000 last_rx_front 2019-12-07T19:33:41.253876+0000 2019-12-07T19:33:41.253+0000 7f2b967b4700 25 osd.6 14 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:41.253160+0000) and older pending ping(s) 2019-12-07T19:33:41.253+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002778139s,0.003513331s]) 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 0x5557e7529700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79cb000 0x5557e7a97600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.253+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6828/36939 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.689336030s delta_ub 18446744073.691222093s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e7af0400 2019-12-07T19:33:41.253+0000 7f2b95fb3700 25 osd.6 14 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253773+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253967+0000 2019-12-07T19:33:41.253+0000 7f2b95fb3700 25 osd.6 14 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:41.253160+0000) and older pending ping(s) 2019-12-07T19:33:41.253+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.7 up_from 0 peer_clock_delta [18446744073.690414494s,18446744073.691222093s]) 2019-12-07T19:33:41.253+0000 7f2b957b2700 10 osd.6 14 ms_handle_authentication new session 0x5557e7517a00 con 0x5557e79cb400 entity osd.4 addr 2019-12-07T19:33:41.253+0000 7f2b957b2700 10 osd.6 14 ms_handle_authentication session 0x5557e7517a00 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.253+0000 7f2b967b4700 10 osd.6 14 ms_handle_authentication new session 0x5557e7517380 con 0x5557e79cb000 entity osd.7 addr 2019-12-07T19:33:41.253+0000 7f2b967b4700 10 osd.6 14 ms_handle_authentication session 0x5557e7517380 osd.7 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x5557e79f4000 0x5557e7a97b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cb400 0x5557e7a97080 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, this connection loses to existing=0x5557e79cbc00 2019-12-07T19:33:41.253+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x5557e79cb000 0x5557e7a97600 crc :-1 s=SESSION_ACCEPTING pgs=8 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, this connection loses to existing=0x5557e79f4000 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cb400 msgr2=0x5557e7a97080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 95 2019-12-07T19:33:41.253+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cb400 msgr2=0x5557e7a97080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.254+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cb400 0x5557e7a97080 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.254+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cb400 0x5557e7a97080 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.254+0000 7f2b957b2700 10 osd.6 14 new session (outgoing) 0x5557e75164e0 con=0x5557e79cbc00 addr=v2:172.21.15.109:6810/36938 2019-12-07T19:33:41.254+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 0x5557e7529700 crc :-1 s=READY pgs=8 cs=0 l=0 rx=0 tx=0).ready entity=osd.4 client_cookie=acae43cf971586cc server_cookie=ee5cf5657875a67a in_seq=0 out_seq=0 2019-12-07T19:33:41.254+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x5557e79cb000 msgr2=0x5557e7a97600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 97 2019-12-07T19:33:41.254+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x5557e79cb000 msgr2=0x5557e7a97600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:41.254+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x5557e79cb000 0x5557e7a97600 crc :-1 s=SESSION_ACCEPTING pgs=8 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:41.254+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x5557e79cb000 0x5557e7a97600 crc :-1 s=SESSION_ACCEPTING pgs=8 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.254+0000 7f2b834f8700 2 osd.6 14 ms_handle_reset con 0x5557e79cb400 session 0x5557e7517a00 2019-12-07T19:33:41.254+0000 7f2b834f8700 10 osd.6 14 ms_handle_connect con 0x5557e79cbc00 2019-12-07T19:33:41.254+0000 7f2b834f8700 2 osd.6 14 ms_handle_reset con 0x5557e79cb000 session 0x5557e7517380 2019-12-07T19:33:41.254+0000 7f2b834f8700 10 osd.6 14 ms_handle_connect con 0x5557e79f4000 2019-12-07T19:33:41.254+0000 7f2b967b4700 10 osd.6 14 new session (outgoing) 0x5557e7517040 con=0x5557e79f4000 addr=v2:172.21.15.109:6826/36939 2019-12-07T19:33:41.254+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x5557e79f4000 0x5557e7a97b80 crc :-1 s=READY pgs=10 cs=0 l=0 rx=0 tx=0).ready entity=osd.7 client_cookie=920af010d0047943 server_cookie=aa504281cd3e39d2 in_seq=0 out_seq=0 2019-12-07T19:33:41.257+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.257+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.706911153s delta_ub 18446744073.704726311s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a200 con 0x5557e7519000 2019-12-07T19:33:41.257+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703958534s,18446744073.704726311s]) 2019-12-07T19:33:41.257+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.706911153s delta_ub 18446744073.704726311s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e74cc400 2019-12-07T19:33:41.257+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.712504235s delta_ub 0.005593082s) v5 -- 0x5557e7a9a600 con 0x5557e7519000 2019-12-07T19:33:41.257+0000 7f2b967b4700 10 osd.6 14 maybe_share_map con 0x5557e79cbc00 v2:172.21.15.109:6810/36938 map epoch 0 -> 13 (as per caller) 2019-12-07T19:33:41.257+0000 7f2b967b4700 10 osd.6 14 send_incremental_map 13 -> 14 to 0x5557e79cbc00 v2:172.21.15.109:6810/36938 2019-12-07T19:33:41.257+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(14..14 src has 1..14) v4 -- 0x5557e6289200 con 0x5557e79cbc00 2019-12-07T19:33:41.257+0000 7f2b967b4700 10 osd.6 14 maybe_share_map con 0x5557e79cbc00 v2:172.21.15.109:6810/36938 map epoch 13 -> 14 (shared) 2019-12-07T19:33:41.257+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703908550s,18446744073.704726311s]) 2019-12-07T19:33:41.257+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.712554219s delta_ub 0.005643066s) v5 -- 0x5557e7a9b400 con 0x5557e74cc400 2019-12-07T19:33:41.271+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.7 172.21.15.109:0/36939 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.691222093s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b200 con 0x5557e748f800 2019-12-07T19:33:41.271+0000 7f2b957b2700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690181445s,18446744073.691222093s]) 2019-12-07T19:33:41.271+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.725990718s delta_ub 0.019370171s) v5 -- 0x5557e74f0000 con 0x5557e748f800 2019-12-07T19:33:41.271+0000 7f2b957b2700 10 osd.6 14 maybe_share_map con 0x5557e79f4000 v2:172.21.15.109:6826/36939 map epoch 0 -> 13 (as per caller) 2019-12-07T19:33:41.271+0000 7f2b957b2700 10 osd.6 14 send_incremental_map 13 -> 14 to 0x5557e79f4000 v2:172.21.15.109:6826/36939 2019-12-07T19:33:41.271+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(14..14 src has 1..14) v4 -- 0x5557e6289200 con 0x5557e79f4000 2019-12-07T19:33:41.271+0000 7f2b957b2700 10 osd.6 14 maybe_share_map con 0x5557e79f4000 v2:172.21.15.109:6826/36939 map epoch 13 -> 14 (shared) 2019-12-07T19:33:41.271+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.7 172.21.15.109:0/36939 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.691222093s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e74cc800 2019-12-07T19:33:41.271+0000 7f2b957b2700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690036622s,18446744073.691222093s]) 2019-12-07T19:33:41.271+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.726135541s delta_ub 0.019514994s) v5 -- 0x5557e74f0000 con 0x5557e74cc800 2019-12-07T19:33:41.307+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.357+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.407+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.457+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.507+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.523+0000 7f2b92fad700 10 osd.6 14 tick 2019-12-07T19:33:41.523+0000 7f2b92fad700 10 osd.6 14 do_waiters -- start 2019-12-07T19:33:41.523+0000 7f2b92fad700 10 osd.6 14 do_waiters -- finish 2019-12-07T19:33:41.523+0000 7f2b92fad700 20 osd.6 14 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:41.557+0000 7f2b8ad07700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258771 2019-12-07T19:33:41.558+0000 7f2b8ad07700 5 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.608+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.650+0000 7f2b9171a700 10 osd.6 14 tick_without_osd_lock 2019-12-07T19:33:41.650+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffde000/0x40000000/0x1680000000, data 0x85ce/0x20000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:41.650+0000 7f2b9171a700 25 osd.6 14 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253538+0000 last_rx_front 2019-12-07T19:33:41.253681+0000 2019-12-07T19:33:41.650+0000 7f2b9171a700 25 osd.6 14 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253456+0000 last_rx_front 2019-12-07T19:33:41.253658+0000 2019-12-07T19:33:41.650+0000 7f2b9171a700 25 osd.6 14 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253703+0000 last_rx_front 2019-12-07T19:33:41.253500+0000 2019-12-07T19:33:41.650+0000 7f2b9171a700 25 osd.6 14 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253895+0000 last_rx_front 2019-12-07T19:33:41.253876+0000 2019-12-07T19:33:41.650+0000 7f2b9171a700 25 osd.6 14 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253617+0000 last_rx_front 2019-12-07T19:33:41.253852+0000 2019-12-07T19:33:41.650+0000 7f2b9171a700 25 osd.6 14 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253810+0000 last_rx_front 2019-12-07T19:33:41.253828+0000 2019-12-07T19:33:41.650+0000 7f2b9171a700 25 osd.6 14 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253773+0000 last_rx_front 2019-12-07T19:33:41.253967+0000 2019-12-07T19:33:41.650+0000 7f2b9171a700 20 osd.6 14 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:41.650+0000 7f2b9171a700 10 osd.6 14 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:41.650+0000 7f2b9171a700 20 osd.6 14 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:41.650+0000 7f2b9171a700 10 osd.6 14 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:41.658+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.708+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.741+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.5 172.21.15.109:0/36936 2 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.207320920s delta_ub 0.012154832s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e62b8c00 con 0x5557e62f4c00 2019-12-07T19:33:41.741+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011447625s,0.012154832s]) 2019-12-07T19:33:41.741+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.195873295s delta_ub 18446744073.698103991s) v5 -- 0x5557e7a9b400 con 0x5557e62f4c00 2019-12-07T19:33:41.741+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.5 172.21.15.109:0/36936 2 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.207320920s delta_ub 0.012154832s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a200 con 0x5557e7518c00 2019-12-07T19:33:41.741+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011402029s,0.012154832s]) 2019-12-07T19:33:41.741+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.195918891s delta_ub 18446744073.698149587s) v5 -- 0x5557e7a9a600 con 0x5557e7518c00 2019-12-07T19:33:41.758+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.808+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.858+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.908+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.958+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.008+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.058+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.109+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.159+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.209+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.259+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.309+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.358+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.807392265s delta_ub 18446744073.704491971s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e74cc400 2019-12-07T19:33:42.358+0000 7f2b95fb3700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703999016s,18446744073.704491971s]) 2019-12-07T19:33:42.358+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.812944865s delta_ub 0.005552600s) v5 -- 0x5557e7a9b400 con 0x5557e74cc400 2019-12-07T19:33:42.358+0000 7f2b95fb3700 10 osd.6 14 maybe_share_map con 0x5557e79cbc00 v2:172.21.15.109:6810/36938 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:42.358+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.807392265s delta_ub 18446744073.704491971s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7519000 2019-12-07T19:33:42.358+0000 7f2b967b4700 20 osd.6 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703954675s,18446744073.704491971s]) 2019-12-07T19:33:42.358+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.812989206s delta_ub 0.005596941s) v5 -- 0x5557e7a9a600 con 0x5557e7519000 2019-12-07T19:33:42.358+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 1 ==== osd_map(15..15 src has 1..15) v4 ==== 380+0+0 (crc 0 0 0) 0x5557e70c3b00 con 0x5557e79cbc00 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 osd.6 14 OSD::ms_dispatch: osd_map(15..15 src has 1..15) v4 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 osd.6 14 do_waiters -- start 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 osd.6 14 do_waiters -- finish 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 osd.6 14 _dispatch 0x5557e70c3b00 osd_map(15..15 src has 1..15) v4 2019-12-07T19:33:42.358+0000 7f2b834f8700 3 osd.6 14 handle_osd_map epochs [15,15], i have 14, src has [1,15] 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 osd.6 14 handle_osd_map got inc map for epoch 15 2019-12-07T19:33:42.358+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3b30e826:::osdmap.14:0# 0x0~0 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e6f54460 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7a91f10 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e6f54150 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x5557e74d90a0 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 = 4456 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 osd.6 14 add_map_bl 14 4456 bytes 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 osd.6 14 got_full_map 15, nothing requested 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 osd.6 14 handle_osd_map pg_num_history pg_num_history(e15 pg_nums {} deleted_pools ) 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps purged_snaps {15={}} 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 osd.6 14 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e15 [1,15] lci=[13,15]) 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7046f00 seq 8 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c923972f:::inc_osdmap.15:0# key 0x7f7fffffffffffffffc92397'/!inc_osdmap.15!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:42.358+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c923972f:::inc_osdmap.15:0# 0x0~150 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1057 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c923972f:::inc_osdmap.15:0# 0x0~150 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~150 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xeb0 on front/back, now 0x0~1000 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7046f00 1 blobs 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:42.358+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x22000~1000/1000,1000,0 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x22000~1000] 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d9340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d92d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d9340 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d92d0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d9340) get_ref 0x0~150 Blob(0x5557e74d9340 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d92d0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d9340) get_ref init 0x1000, 1000 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~150: 0x0~150 Blob(0x5557e74d9340 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x150) SharedBlob(0x5557e74d92d0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e74d92e8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x150 2019-12-07T19:33:42.358+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556108) dirty_range mark inline shard dirty 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c923972f:::inc_osdmap.15:0# 0x0~150 = 0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3630e826:::osdmap.15:0# key 0x7f7fffffffffffffff3630e8'&!osdmap.15!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:42.359+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3630e826:::osdmap.15:0# 0x0~1168 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1058 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3630e826:::osdmap.15:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~168 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7046f00 2 blobs 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x23000~2000/1000,2000,0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x23000~2000] 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d91f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d91f0 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8ee0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d91f0) get_ref 0x0~1000 Blob(0x5557e74d91f0 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8ee0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d91f0) get_ref init 0x1000, 1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e74d91f0 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e74d8ee0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e74d8ef8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d9b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d9b90 blob([!~1000,0x24000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9c00 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d9b90) get_ref 0x1000~168 Blob(0x5557e74d9b90 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9c00 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d9b90) get_ref init 0x2000, 1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x5557e74d9b90 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x5557e74d9c00 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e74d9c18 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1168 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556388) dirty_range mark inline shard dirty 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3630e826:::osdmap.15:0# 0x0~1168 = 0 2019-12-07T19:33:42.359+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~12 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7046f00 0 blobs 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:42.359+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:42.359+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:42.359+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7046f00 0 blobs 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 22253 (4 ios * 4000 + 6253 bytes) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7556000,0x5557e7556280 shared_blobs 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556108) update #-1:c923972f:::inc_osdmap.15:0# 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556108) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c923972f:::inc_osdmap.15:0# is 46 (22 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556388) update #-1:3630e826:::osdmap.15:0# 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556388) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3630e826:::osdmap.15:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[22000~3000] released 0x[] 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 freelist allocate 0x22000~3000 2019-12-07T19:33:42.359+0000 7f2b834f8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:42.359+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7046f00 2019-12-07T19:33:42.359+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 aio_wait 2019-12-07T19:33:42.359+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:42.359+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8787 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7556000 had 1 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7556280 had 1 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8810 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8810 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x82be~226a to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x82be 0x82be~226a to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x82be 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x2be 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x528 and padding block with 0xad8 2019-12-07T19:33:42.359+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0xa528 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:42.359+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000226092s (0.000052178s flush + 0.000173914s kv commit) 2019-12-07T19:33:42.359+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:42.359+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7046f00 osr 0x5557e76ce500 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79edb80) _discard drop 5 0x1b000~1000 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6 14 _committed_osd_maps 15..15 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6 14 advance to epoch 15 (<= last 15 <= newest_map 15) 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79edb80) prepare_write seq 6 0x1b000~1000 crc 1e19645a 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79edb80) _discard drop 5 0x2000~1000 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79edb80) prepare_write seq 6 0x2000~1000 crc a4bcceb5 2019-12-07T19:33:42.359+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:42.359+0000 7f2b85cfd700 7 osd.6 15 consume_map version 15 2019-12-07T19:33:42.359+0000 7f2b85cfd700 20 osd.6 15 prune_pg_created 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6:0.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6:1.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6:2.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6:3.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6:4.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6:5.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6:6.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6:7.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7f2b85cfd700 10 osd.6 15 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:42.359+0000 7f2b85cfd700 7 osd.6 15 activate_map version 15 2019-12-07T19:33:42.359+0000 7f2b7ace7700 20 osd.6 15 agent_entry empty queue 2019-12-07T19:33:42.409+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.459+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.509+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.559+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:33:42.559+0000 7f2b8ad07700 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock osr 0x5557e76ce500 2 ios pending 2019-12-07T19:33:42.559+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock seq 6 0x2000~1000 2019-12-07T19:33:42.559+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock write 0x2000~1000 crc a4bcceb5 2019-12-07T19:33:42.559+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock seq 6 0x1b000~1000 2019-12-07T19:33:42.559+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock write 0x1b000~1000 crc 1e19645a 2019-12-07T19:33:42.559+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.559+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_aio_finish osr 0x5557e76ce500 2019-12-07T19:33:42.559+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_aio_finish dequeueing 2019-12-07T19:33:42.561+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:42.561+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:42.561+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:42.561+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:42.610+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.629+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:42.629+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:42.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253538+0000 last_rx_front 2019-12-07T19:33:41.253681+0000 2019-12-07T19:33:42.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253456+0000 last_rx_front 2019-12-07T19:33:41.253658+0000 2019-12-07T19:33:42.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253703+0000 last_rx_front 2019-12-07T19:33:41.253500+0000 2019-12-07T19:33:42.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253895+0000 last_rx_front 2019-12-07T19:33:41.253876+0000 2019-12-07T19:33:42.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253617+0000 last_rx_front 2019-12-07T19:33:41.253852+0000 2019-12-07T19:33:42.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253810+0000 last_rx_front 2019-12-07T19:33:41.253828+0000 2019-12-07T19:33:42.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253773+0000 last_rx_front 2019-12-07T19:33:41.253967+0000 2019-12-07T19:33:42.629+0000 7f2b9171a700 20 osd.6 15 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:42.629+0000 7f2b9171a700 20 osd.6 15 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:42.629+0000 7f2b9171a700 20 osd.6 15 scrub_load_below_threshold loadavg per cpu 0.0725 < max 5 = yes 2019-12-07T19:33:42.629+0000 7f2b9171a700 20 osd.6 15 sched_scrub load_is_low=1 2019-12-07T19:33:42.629+0000 7f2b9171a700 20 osd.6 15 sched_scrub done 2019-12-07T19:33:42.629+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:42.629+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:42.629+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:42.660+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.710+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.760+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.810+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.858+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.307750221s delta_ub 18446744073.704494230s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e62b8c00 con 0x5557e74cc400 2019-12-07T19:33:42.858+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704094986s,18446744073.704494230s]) 2019-12-07T19:33:42.858+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.313206851s delta_ub 0.005456630s) v5 -- 0x5557e7a9b400 con 0x5557e74cc400 2019-12-07T19:33:42.858+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.307750221s delta_ub 18446744073.704494230s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a200 con 0x5557e7519000 2019-12-07T19:33:42.858+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704072541s,18446744073.704494230s]) 2019-12-07T19:33:42.858+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.313229296s delta_ub 0.005479075s) v5 -- 0x5557e7a9a600 con 0x5557e7519000 2019-12-07T19:33:42.860+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.910+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.960+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.010+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.061+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.111+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.161+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.211+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.261+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.311+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.361+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.412+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.462+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.512+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.521+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:43.521+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:43.521+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:43.521+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:43.562+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.607+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:43.607+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:43.607+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253538+0000 last_rx_front 2019-12-07T19:33:41.253681+0000 2019-12-07T19:33:43.607+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253456+0000 last_rx_front 2019-12-07T19:33:41.253658+0000 2019-12-07T19:33:43.607+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253703+0000 last_rx_front 2019-12-07T19:33:41.253500+0000 2019-12-07T19:33:43.607+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253895+0000 last_rx_front 2019-12-07T19:33:41.253876+0000 2019-12-07T19:33:43.607+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253617+0000 last_rx_front 2019-12-07T19:33:41.253852+0000 2019-12-07T19:33:43.607+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253810+0000 last_rx_front 2019-12-07T19:33:41.253828+0000 2019-12-07T19:33:43.607+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253773+0000 last_rx_front 2019-12-07T19:33:41.253967+0000 2019-12-07T19:33:43.607+0000 7f2b9171a700 20 osd.6 15 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:43.607+0000 7f2b9171a700 20 osd.6 15 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:43.607+0000 7f2b9171a700 20 osd.6 15 scrub_load_below_threshold loadavg per cpu 0.0725 < max 5 = yes 2019-12-07T19:33:43.607+0000 7f2b9171a700 20 osd.6 15 sched_scrub load_is_low=1 2019-12-07T19:33:43.607+0000 7f2b9171a700 20 osd.6 15 sched_scrub done 2019-12-07T19:33:43.607+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:43.607+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:43.607+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:43.612+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.662+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.712+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.762+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.812+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.862+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.913+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.963+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.013+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.041+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.5 172.21.15.109:0/36936 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.507816348s delta_ub 0.011910080s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e62f4c00 2019-12-07T19:33:44.041+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011609710s,0.011910080s]) 2019-12-07T19:33:44.041+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.496206638s delta_ub 18446744073.697941906s) v5 -- 0x5557e7a9b400 con 0x5557e62f4c00 2019-12-07T19:33:44.041+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.5 172.21.15.109:0/36936 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.507816348s delta_ub 0.011910080s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7518c00 2019-12-07T19:33:44.041+0000 7f2b95fb3700 10 osd.6 15 maybe_share_map con 0x5557e79ca000 v2:172.21.15.109:6802/36936 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.041+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011589918s,0.011910080s]) 2019-12-07T19:33:44.041+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.496226430s delta_ub 18446744073.697961698s) v5 -- 0x5557e7a9a600 con 0x5557e7518c00 2019-12-07T19:33:44.063+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.113+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.163+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.213+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.263+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.313+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.363+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.414+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.464+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.493+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:44.493+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:44.493+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:44.493+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:44.514+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.564+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.614+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.646+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:44.646+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:44.646+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253538+0000 last_rx_front 2019-12-07T19:33:41.253681+0000 2019-12-07T19:33:44.646+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253456+0000 last_rx_front 2019-12-07T19:33:41.253658+0000 2019-12-07T19:33:44.646+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253703+0000 last_rx_front 2019-12-07T19:33:41.253500+0000 2019-12-07T19:33:44.646+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253895+0000 last_rx_front 2019-12-07T19:33:41.253876+0000 2019-12-07T19:33:44.646+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253617+0000 last_rx_front 2019-12-07T19:33:41.253852+0000 2019-12-07T19:33:44.646+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253810+0000 last_rx_front 2019-12-07T19:33:41.253828+0000 2019-12-07T19:33:44.646+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253773+0000 last_rx_front 2019-12-07T19:33:41.253967+0000 2019-12-07T19:33:44.646+0000 7f2b9171a700 20 osd.6 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:44.646+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:44.646+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:44.646+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:44.664+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.714+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.752+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.0 172.21.15.131:0/37079 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.000490236s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a200 con 0x5557e7518800 2019-12-07T19:33:44.752+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709489586s,0.000490236s]) 2019-12-07T19:33:44.752+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207194526s delta_ub 0.000062030s) v5 -- 0x5557e7a9a600 con 0x5557e7518800 2019-12-07T19:33:44.752+0000 7f2b967b4700 10 osd.6 15 maybe_share_map con 0x5557e79cb800 v2:172.21.15.131:6822/37079 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.752+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.0 172.21.15.131:0/37079 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.000490236s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7518000 2019-12-07T19:33:44.752+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709379320s,0.000490236s]) 2019-12-07T19:33:44.752+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207304792s delta_ub 0.000172296s) v5 -- 0x5557e7a9a600 con 0x5557e7518000 2019-12-07T19:33:44.764+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.815+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.865+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.915+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.965+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.015+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.065+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.115+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.165+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.215+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.266+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.316+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.350+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.807969424s delta_ub 0.003513331s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a200 con 0x5557e7519400 2019-12-07T19:33:45.350+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002820940s,0.003513331s]) 2019-12-07T19:33:45.350+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.805148484s delta_ub 18446744073.706730676s) v5 -- 0x5557e7a9a600 con 0x5557e7519400 2019-12-07T19:33:45.350+0000 7f2b967b4700 10 osd.6 15 maybe_share_map con 0x5557e79ca800 v2:172.21.15.131:6812/37076 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:45.350+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.1 172.21.15.131:0/37078 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.807380414s delta_ub 0.002772195s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7518400 2019-12-07T19:33:45.350+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002172341s,0.002772195s]) 2019-12-07T19:33:45.350+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.805208073s delta_ub 18446744073.707379275s) v5 -- 0x5557e7a9a600 con 0x5557e7518400 2019-12-07T19:33:45.350+0000 7f2b967b4700 10 osd.6 15 maybe_share_map con 0x5557e79cac00 v2:172.21.15.131:6823/37078 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:45.350+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.807969424s delta_ub 0.003513331s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a200 con 0x5557e74cd800 2019-12-07T19:33:45.350+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002623202s,0.003513331s]) 2019-12-07T19:33:45.350+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.805346222s delta_ub 18446744073.706928414s) v5 -- 0x5557e7a9a600 con 0x5557e74cd800 2019-12-07T19:33:45.350+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.1 172.21.15.131:0/37078 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.807380414s delta_ub 0.002772195s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e74cdc00 2019-12-07T19:33:45.350+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001987481s,0.002772195s]) 2019-12-07T19:33:45.350+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.805392933s delta_ub 18446744073.707564135s) v5 -- 0x5557e7a9a600 con 0x5557e74cdc00 2019-12-07T19:33:45.366+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.371+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.7 172.21.15.109:0/36939 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.690683952s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e748f800 2019-12-07T19:33:45.371+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690295955s,18446744073.690683952s]) 2019-12-07T19:33:45.371+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.826250921s delta_ub 0.019255661s) v5 -- 0x5557e74f0000 con 0x5557e748f800 2019-12-07T19:33:45.371+0000 7f2b957b2700 10 osd.6 15 maybe_share_map con 0x5557e79f4000 v2:172.21.15.109:6826/36939 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:45.371+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.7 172.21.15.109:0/36939 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.690683952s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b200 con 0x5557e74cc800 2019-12-07T19:33:45.371+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690218059s,18446744073.690683952s]) 2019-12-07T19:33:45.371+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.826328817s delta_ub 0.019333557s) v5 -- 0x5557e74f0000 con 0x5557e74cc800 2019-12-07T19:33:45.416+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.466+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.483+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:45.483+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:45.483+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:45.483+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:45.516+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.520+0000 7f2b83cf9700 20 osd.6 15 reports for 0 queries 2019-12-07T19:33:45.520+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.6 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x5557e79c4380 con 0x5557e7a48000 2019-12-07T19:33:45.520+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x5557e79c4700 con 0x5557e7a48000 2019-12-07T19:33:45.566+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:45.566+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.616+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.642+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:45.642+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:45.642+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253538+0000 last_rx_front 2019-12-07T19:33:41.253681+0000 2019-12-07T19:33:45.642+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253456+0000 last_rx_front 2019-12-07T19:33:41.253658+0000 2019-12-07T19:33:45.642+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253703+0000 last_rx_front 2019-12-07T19:33:41.253500+0000 2019-12-07T19:33:45.642+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253895+0000 last_rx_front 2019-12-07T19:33:41.253876+0000 2019-12-07T19:33:45.642+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253617+0000 last_rx_front 2019-12-07T19:33:41.253852+0000 2019-12-07T19:33:45.642+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253810+0000 last_rx_front 2019-12-07T19:33:41.253828+0000 2019-12-07T19:33:45.642+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:41.253160+0000 last_rx_back 2019-12-07T19:33:41.253773+0000 last_rx_front 2019-12-07T19:33:41.253967+0000 2019-12-07T19:33:45.642+0000 7f2b9171a700 20 osd.6 15 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:45.642+0000 7f2b9171a700 20 osd.6 15 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:45.642+0000 7f2b9171a700 20 osd.6 15 scrub_load_below_threshold loadavg per cpu 0.06625 < max 5 = yes 2019-12-07T19:33:45.642+0000 7f2b9171a700 20 osd.6 15 sched_scrub load_is_low=1 2019-12-07T19:33:45.642+0000 7f2b9171a700 20 osd.6 15 sched_scrub done 2019-12-07T19:33:45.642+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:45.642+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:45.642+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:45.662+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e79f4400 0x5557e7a98100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:45.662+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e79f4400 0x5557e7a98100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:45.662+0000 7f2b957b2700 10 osd.6 15 ms_handle_authentication new session 0x5557e7517ba0 con 0x5557e79f4400 entity client.admin addr 2019-12-07T19:33:45.662+0000 7f2b957b2700 10 osd.6 15 ms_handle_authentication session 0x5557e7517ba0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:45.662+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/1689863392 conn(0x5557e79f4400 0x5557e7a98100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4321 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:45.662+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4321 172.21.15.131:0/1689863392 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x5557e6281a40 con 0x5557e79f4400 2019-12-07T19:33:45.662+0000 7f2b88017700 20 osd.6 15 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:45.662+0000 7f2b88017700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:45.662+0000 7f2b88017700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:45.662+0000 7f2b88017700 20 osd.6 15 _dispatch 0x5557e6281a40 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:45.662+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/1689863392 -- command_reply(tid 1: 0 ) v1 -- 0x5557e627e380 con 0x5557e79f4400 2019-12-07T19:33:45.666+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.683+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4321 172.21.15.131:0/1689863392 2 ==== command(tid 2: {"prefix": "flush_pg_stats"}) v1 ==== 52+0+0 (crc 0 0 0) 0x5557e6281e00 con 0x5557e79f4400 2019-12-07T19:33:45.683+0000 7f2b88017700 20 osd.6 15 OSD::ms_dispatch: command(tid 2: {"prefix": "flush_pg_stats"}) v1 2019-12-07T19:33:45.683+0000 7f2b88017700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:45.683+0000 7f2b88017700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:45.683+0000 7f2b88017700 20 osd.6 15 _dispatch 0x5557e6281e00 command(tid 2: {"prefix": "flush_pg_stats"}) v1 2019-12-07T19:33:45.683+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x5557e79c5180 con 0x5557e7a48000 2019-12-07T19:33:45.683+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/1689863392 -- command_reply(tid 2: 0 ) v1 -- 0x5557e627f340 con 0x5557e79f4400 2019-12-07T19:33:45.685+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/1689863392 conn(0x5557e79f4400 msgr2=0x5557e7a98100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:45.685+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/1689863392 conn(0x5557e79f4400 msgr2=0x5557e7a98100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:45.685+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/1689863392 conn(0x5557e79f4400 0x5557e7a98100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:45.685+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/1689863392 conn(0x5557e79f4400 0x5557e7a98100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:45.685+0000 7f2b88017700 2 osd.6 15 ms_handle_reset con 0x5557e79f4400 session 0x5557e7517ba0 2019-12-07T19:33:45.716+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.766+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.817+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.867+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.917+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.953+0000 7f2b724d6700 5 osd.6 15 heartbeat osd_stat(store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,7] op hist []) 2019-12-07T19:33:45.953+0000 7f2b724d6700 20 osd.6 15 check_full_status cur ratio 1.56826e-06, physical ratio 1.56826e-06, new state none 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.000172296s) v5 -- 0x5557e7a9bc00 con 0x5557e7a49000 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.000172296s) v5 -- 0x5557e7a9be00 con 0x5557e7a49400 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 18446744073.707564135s) v5 -- 0x5557e758c000 con 0x5557e7af0800 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 18446744073.707564135s) v5 -- 0x5557e758c200 con 0x5557e7af0c00 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 18446744073.692627271s) v5 -- 0x5557e758c400 con 0x5557e7af1000 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 18446744073.692627271s) v5 -- 0x5557e758c600 con 0x5557e7af1400 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 18446744073.706928414s) v5 -- 0x5557e758c800 con 0x5557e7af1800 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 18446744073.706928414s) v5 -- 0x5557e758ca00 con 0x5557e7af1c00 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.005479075s) v5 -- 0x5557e758cc00 con 0x5557e748e000 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.005479075s) v5 -- 0x5557e7a9b800 con 0x5557e748e400 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 18446744073.697961698s) v5 -- 0x5557e7a9a600 con 0x5557e7a49800 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 18446744073.697961698s) v5 -- 0x5557e7a9a200 con 0x5557e7a49c00 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.019333557s) v5 -- 0x5557e7a9be00 con 0x5557e7af0000 2019-12-07T19:33:45.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.019333557s) v5 -- 0x5557e7a9b200 con 0x5557e7af0400 2019-12-07T19:33:45.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6814/36938 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.402967776s delta_ub 18446744073.704468895s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758ca00 con 0x5557e748e000 2019-12-07T19:33:45.953+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253617+0000 -> 2019-12-07T19:33:45.953801+0000 last_rx_front 2019-12-07T19:33:41.253852+0000 2019-12-07T19:33:45.953+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704212163s,18446744073.704468895s]) 2019-12-07T19:33:45.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6804/36936 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.420098701s delta_ub 0.012048204s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758ca00 con 0x5557e7a49c00 2019-12-07T19:33:45.953+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253810+0000 last_rx_front 2019-12-07T19:33:41.253828+0000 -> 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:45.953+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011750643s,0.012048204s]) 2019-12-07T19:33:45.953+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6816/37076 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.411338430s delta_ub 0.003287933s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758dc00 con 0x5557e7af1800 2019-12-07T19:33:45.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6806/37077 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.425539963s delta_ub 0.017489466s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758ca00 con 0x5557e7af1400 2019-12-07T19:33:45.953+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253895+0000 -> 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:41.253876+0000 2019-12-07T19:33:45.953+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002966616s,0.003287933s]) 2019-12-07T19:33:45.953+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253703+0000 last_rx_front 2019-12-07T19:33:41.253500+0000 -> 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:45.953+0000 7f2b957b2700 10 osd.6 15 maybe_share_map con 0x5557e7519800 v2:172.21.15.131:6804/37077 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:45.953+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017152977s,0.017489466s]) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6808/37077 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.425574575s delta_ub 0.017524078s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e7af1000 2019-12-07T19:33:45.953+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6827/37078 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.410757862s delta_ub 0.002707365s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758dc00 con 0x5557e7af0c00 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253703+0000 -> 2019-12-07T19:33:45.953897+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:45.953545+0000) and older pending ping(s) 2019-12-07T19:33:45.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6826/37079 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408511831s delta_ub 0.000461334s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758ca00 con 0x5557e7a49400 2019-12-07T19:33:45.953+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017171753s,0.017524078s]) 2019-12-07T19:33:45.953+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253456+0000 last_rx_front 2019-12-07T19:33:41.253658+0000 -> 2019-12-07T19:33:45.953915+0000 2019-12-07T19:33:45.953+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002337307s,0.002707365s]) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6814/37076 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.411358634s delta_ub 0.003308137s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e7af1c00 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:41.253876+0000 -> 2019-12-07T19:33:45.953925+0000 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:45.953545+0000) and older pending ping(s) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002927844s,0.003308137s]) 2019-12-07T19:33:45.953+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253538+0000 last_rx_front 2019-12-07T19:33:41.253681+0000 -> 2019-12-07T19:33:45.953936+0000 2019-12-07T19:33:45.953+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.000070424s,0.000461334s]) 2019-12-07T19:33:45.953+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6812/36938 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.403028583s delta_ub 18446744073.704529702s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758dc00 con 0x5557e748e400 2019-12-07T19:33:45.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6828/36939 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.389243407s delta_ub 18446744073.690744526s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e7af0400 2019-12-07T19:33:45.953+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953801+0000 last_rx_front 2019-12-07T19:33:41.253852+0000 -> 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:45.953+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:45.953545+0000) and older pending ping(s) 2019-12-07T19:33:45.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6831/37078 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.410823918s delta_ub 0.002773421s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758ca00 con 0x5557e7af0800 2019-12-07T19:33:45.953+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704131118s,18446744073.704529702s]) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253773+0000 last_rx_front 2019-12-07T19:33:41.253967+0000 -> 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:45.953+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690325787s,18446744073.690744526s]) 2019-12-07T19:33:45.953+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253456+0000 -> 2019-12-07T19:33:45.953973+0000 last_rx_front 2019-12-07T19:33:45.953915+0000 2019-12-07T19:33:45.953+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6830/36939 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.389274329s delta_ub 18446744073.690775448s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758dc00 con 0x5557e7af0000 2019-12-07T19:33:45.953+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:45.953545+0000) and older pending ping(s) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6830/37079 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408558652s delta_ub 0.000508155s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e7a49000 2019-12-07T19:33:45.953+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002345083s,0.002773421s]) 2019-12-07T19:33:45.953+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253773+0000 -> 2019-12-07T19:33:45.953989+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:45.953+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:45.953545+0000) and older pending ping(s) 2019-12-07T19:33:45.953+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690331201s,18446744073.690775448s]) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253538+0000 -> 2019-12-07T19:33:45.954007+0000 last_rx_front 2019-12-07T19:33:45.953936+0000 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:45.953545+0000) and older pending ping(s) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.000045973s,0.000508155s]) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6806/36936 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.420132451s delta_ub 0.012081954s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e7a49800 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:41.253810+0000 -> 2019-12-07T19:33:45.954030+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:45.953+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:45.953545+0000) and older pending ping(s) 2019-12-07T19:33:45.953+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011596682s,0.012081954s]) 2019-12-07T19:33:45.967+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.017+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.067+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.117+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.167+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.217+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.268+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.318+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.358+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.808045243s delta_ub 18446744073.704529702s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758dc00 con 0x5557e7519000 2019-12-07T19:33:46.358+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704065952s,18446744073.704529702s]) 2019-12-07T19:33:46.358+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.808045243s delta_ub 18446744073.704529702s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e74cc400 2019-12-07T19:33:46.358+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.813530907s delta_ub 0.005485664s) v5 -- 0x5557e758da00 con 0x5557e7519000 2019-12-07T19:33:46.358+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704043273s,18446744073.704529702s]) 2019-12-07T19:33:46.358+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.813553586s delta_ub 0.005508343s) v5 -- 0x5557e7a9b200 con 0x5557e74cc400 2019-12-07T19:33:46.368+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.418+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.447+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:46.447+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:46.447+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:46.447+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:46.450+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.908320123s delta_ub 0.003308137s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e7519400 2019-12-07T19:33:46.450+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002809912s,0.003308137s]) 2019-12-07T19:33:46.450+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.905510211s delta_ub 18446744073.706741704s) v5 -- 0x5557e758da00 con 0x5557e7519400 2019-12-07T19:33:46.450+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.908320123s delta_ub 0.003308137s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758dc00 con 0x5557e74cd800 2019-12-07T19:33:46.450+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002757955s,0.003308137s]) 2019-12-07T19:33:46.450+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.905562168s delta_ub 18446744073.706793661s) v5 -- 0x5557e758da00 con 0x5557e74cd800 2019-12-07T19:33:46.468+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.518+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.536+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.2 172.21.15.131:0/37077 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 10.007789311s delta_ub 0.017524078s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e74cd400 2019-12-07T19:33:46.536+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.2 172.21.15.131:0/37077 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 10.007789311s delta_ub 0.017524078s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758ca00 con 0x5557e74ccc00 2019-12-07T19:33:46.536+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016995451s,0.017524078s]) 2019-12-07T19:33:46.536+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 9.990793860s delta_ub 18446744073.692556165s) v5 -- 0x5557e758da00 con 0x5557e74cd400 2019-12-07T19:33:46.536+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016928995s,0.017524078s]) 2019-12-07T19:33:46.536+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 9.990860316s delta_ub 18446744073.692622621s) v5 -- 0x5557e758cc00 con 0x5557e74ccc00 2019-12-07T19:33:46.552+0000 7f2b87015700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- log(2 entries from seq 1 at 2019-12-07T19:33:37.564613+0000) v1 -- 0x5557e74ca240 con 0x5557e6f87000 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:46.568+0000 7f2b8ad07700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258772 2019-12-07T19:33:46.568+0000 7f2b8ad07700 5 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.618+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.655+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:46.655+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:46.655+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954007+0000 last_rx_front 2019-12-07T19:33:45.953936+0000 2019-12-07T19:33:46.655+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953973+0000 last_rx_front 2019-12-07T19:33:45.953915+0000 2019-12-07T19:33:46.655+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953897+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:46.655+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:45.953925+0000 2019-12-07T19:33:46.655+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953801+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:46.655+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954030+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:46.655+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953989+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:46.655+0000 7f2b9171a700 20 osd.6 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:46.655+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:46.655+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:46.655+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:46.669+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.719+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.769+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.819+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.869+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.919+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.969+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.019+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.070+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.120+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.170+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.218+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 15 ==== log(last 2) v1 ==== 28+0+0 (crc 0 0 0) 0x5557e6281a40 con 0x5557e6f87000 2019-12-07T19:33:47.220+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.270+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.320+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.370+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.420+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.470+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.477+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:47.477+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:47.477+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:47.477+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:47.520+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.571+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.621+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.624+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:47.624+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:47.624+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954007+0000 last_rx_front 2019-12-07T19:33:45.953936+0000 2019-12-07T19:33:47.624+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953973+0000 last_rx_front 2019-12-07T19:33:45.953915+0000 2019-12-07T19:33:47.624+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953897+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:47.624+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:45.953925+0000 2019-12-07T19:33:47.624+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953801+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:47.624+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954030+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:47.624+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953989+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:47.624+0000 7f2b9171a700 20 osd.6 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:47.624+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:47.624+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:47.624+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:47.671+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.721+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.771+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.821+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.871+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.921+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.971+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.021+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.072+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.122+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.608668265s delta_ub 0.003133130s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758dc00 con 0x5557e7519400 2019-12-07T19:33:48.151+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002705021s,0.003133130s]) 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.605963244s delta_ub 18446744073.706846595s) v5 -- 0x5557e758da00 con 0x5557e7519400 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.608668265s delta_ub 0.003133130s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f0800 con 0x5557e74cd800 2019-12-07T19:33:48.151+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002607460s,0.003133130s]) 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.606060805s delta_ub 18446744073.706944156s) v5 -- 0x5557e758da00 con 0x5557e74cd800 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37076 conn(0x5557e74cd800 msgr2=0x5557e74c0680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 84 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37076 conn(0x5557e74cd800 msgr2=0x5557e74c0680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37076 conn(0x5557e74cd800 0x5557e74c0680 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37076 conn(0x5557e74cd800 0x5557e74c0680 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:48.151+0000 7f2b804f2700 20 osd.6 15 heartbeat_reset con 0x5557e74cd800 s 0x5557e75161a0 2019-12-07T19:33:48.151+0000 7f2b804f2700 10 osd.6 15 heartbeat_reset closing (old) failed hb con 0x5557e74cd800 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37076 conn(0x5557e7519400 msgr2=0x5557e7527080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 88 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37076 conn(0x5557e7519400 msgr2=0x5557e7527080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37076 conn(0x5557e7519400 0x5557e7527080 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37076 conn(0x5557e7519400 0x5557e7527080 crc :-1 s=READY pgs=10 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:48.151+0000 7f2b7f4f0700 20 osd.6 15 heartbeat_reset con 0x5557e7519400 s 0x5557e7516ea0 2019-12-07T19:33:48.151+0000 7f2b7f4f0700 10 osd.6 15 heartbeat_reset closing (old) failed hb con 0x5557e7519400 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e79f4800 0x5557e7a98680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e79f4c00 0x5557e7a96b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> conn(0x5557e79f4800 0x5557e7a98680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:48.151+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> conn(0x5557e79f4c00 0x5557e7a96b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:48.151+0000 7f2b967b4700 10 osd.6 15 ms_handle_authentication new session 0x5557e75584e0 con 0x5557e79f4800 entity osd.3 addr 2019-12-07T19:33:48.151+0000 7f2b967b4700 10 osd.6 15 ms_handle_authentication session 0x5557e75584e0 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:48.151+0000 7f2b957b2700 10 osd.6 15 ms_handle_authentication new session 0x5557e7517d40 con 0x5557e79f4c00 entity osd.3 addr 2019-12-07T19:33:48.151+0000 7f2b957b2700 10 osd.6 15 ms_handle_authentication session 0x5557e7517d40 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:48.151+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] >> 172.21.15.131:0/37076 conn(0x5557e79f4800 0x5557e7a98680 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:48.151+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] >> 172.21.15.131:0/37076 conn(0x5557e79f4c00 0x5557e7a96b00 crc :-1 s=READY pgs=12 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:48.172+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.222+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.272+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.322+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.372+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.422+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.437+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:48.437+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:48.437+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:48.437+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:48.472+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.522+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.573+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:48.573+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.623+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.672+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:48.672+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:48.672+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954007+0000 last_rx_front 2019-12-07T19:33:45.953936+0000 2019-12-07T19:33:48.672+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953973+0000 last_rx_front 2019-12-07T19:33:45.953915+0000 2019-12-07T19:33:48.672+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953897+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:48.672+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:45.953925+0000 2019-12-07T19:33:48.672+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953801+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:48.672+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954030+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:48.672+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953989+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:48.672+0000 7f2b9171a700 20 osd.6 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:48.672+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:48.672+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:48.672+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:48.673+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.723+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.773+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.823+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.852+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.0 172.21.15.131:0/37079 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.000508155s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c800 con 0x5557e7518800 2019-12-07T19:33:48.852+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709493720s,0.000508155s]) 2019-12-07T19:33:48.852+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307478930s delta_ub 0.000057896s) v5 -- 0x5557e758c000 con 0x5557e7518800 2019-12-07T19:33:48.852+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.0 172.21.15.131:0/37079 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.000508155s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7518000 2019-12-07T19:33:48.852+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709337632s,0.000508155s]) 2019-12-07T19:33:48.852+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307635018s delta_ub 0.000213984s) v5 -- 0x5557e758c000 con 0x5557e7518000 2019-12-07T19:33:48.871+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.7 172.21.15.109:0/36939 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.690775448s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758cc00 con 0x5557e748f800 2019-12-07T19:33:48.871+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690225316s,18446744073.690775448s]) 2019-12-07T19:33:48.871+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.326655872s delta_ub 0.019326300s) v5 -- 0x5557e62b8c00 con 0x5557e748f800 2019-12-07T19:33:48.872+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.7 172.21.15.109:0/36939 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.690775448s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e74cc800 2019-12-07T19:33:48.872+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690139218s,18446744073.690775448s]) 2019-12-07T19:33:48.872+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.326741970s delta_ub 0.019412398s) v5 -- 0x5557e62b8c00 con 0x5557e74cc800 2019-12-07T19:33:48.873+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.923+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.973+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.023+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.074+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.124+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.174+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.224+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.259+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.708361738s delta_ub 18446744073.704424519s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c800 con 0x5557e7519000 2019-12-07T19:33:49.259+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704060187s,18446744073.704424519s]) 2019-12-07T19:33:49.259+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.713853167s delta_ub 0.005491429s) v5 -- 0x5557e758c000 con 0x5557e7519000 2019-12-07T19:33:49.259+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.708361738s delta_ub 18446744073.704424519s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a600 con 0x5557e74cc400 2019-12-07T19:33:49.259+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704034046s,18446744073.704424519s]) 2019-12-07T19:33:49.259+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.713879308s delta_ub 0.005517570s) v5 -- 0x5557e7a9b200 con 0x5557e74cc400 2019-12-07T19:33:49.274+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.324+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.374+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.424+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.474+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.480+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:49.480+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:49.480+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:49.480+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:49.525+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.575+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.623+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:49.623+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:49.623+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954007+0000 last_rx_front 2019-12-07T19:33:45.953936+0000 2019-12-07T19:33:49.623+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953973+0000 last_rx_front 2019-12-07T19:33:45.953915+0000 2019-12-07T19:33:49.623+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953897+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:49.623+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:45.953925+0000 2019-12-07T19:33:49.623+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953801+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:49.623+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954030+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:49.623+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953989+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:49.623+0000 7f2b9171a700 20 osd.6 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:49.623+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:49.623+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:49.623+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:49.625+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.675+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.725+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.775+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.825+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.851+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.309014358s delta_ub 0.003180510s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758cc00 con 0x5557e79f4c00 2019-12-07T19:33:49.851+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.309014358s delta_ub 0.003180510s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e79f4800 2019-12-07T19:33:49.851+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002732279s,0.003180510s]) 2019-12-07T19:33:49.851+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.306282079s delta_ub 18446744073.706819337s) v5 -- 0x5557e62b8c00 con 0x5557e79f4c00 2019-12-07T19:33:49.851+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002687130s,0.003180510s]) 2019-12-07T19:33:49.851+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.306327228s delta_ub 18446744073.706864486s) v5 -- 0x5557e758c000 con 0x5557e79f4800 2019-12-07T19:33:49.875+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.925+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.941+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.5 172.21.15.109:0/36936 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.408188910s delta_ub 0.012081954s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a800 con 0x5557e62f4c00 2019-12-07T19:33:49.941+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011576693s,0.012081954s]) 2019-12-07T19:33:49.941+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.5 172.21.15.109:0/36936 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.408188910s delta_ub 0.012081954s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c800 con 0x5557e7518c00 2019-12-07T19:33:49.941+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.396612217s delta_ub 18446744073.697974923s) v5 -- 0x5557e7a9b200 con 0x5557e62f4c00 2019-12-07T19:33:49.941+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011554891s,0.012081954s]) 2019-12-07T19:33:49.941+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.396634019s delta_ub 18446744073.697996725s) v5 -- 0x5557e758c000 con 0x5557e7518c00 2019-12-07T19:33:49.976+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.026+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.076+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.126+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.176+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.226+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.276+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.326+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.376+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.426+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.449+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:50.449+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:50.449+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:50.449+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:50.477+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.521+0000 7f2b83cf9700 20 osd.6 15 reports for 0 queries 2019-12-07T19:33:50.521+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.6 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x5557e79c4a80 con 0x5557e7a48000 2019-12-07T19:33:50.521+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x5557e79c4380 con 0x5557e7a48000 2019-12-07T19:33:50.526+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.577+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.627+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.649+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:50.649+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:50.649+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954007+0000 last_rx_front 2019-12-07T19:33:45.953936+0000 2019-12-07T19:33:50.649+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953973+0000 last_rx_front 2019-12-07T19:33:45.953915+0000 2019-12-07T19:33:50.649+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953897+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:50.649+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:45.953925+0000 2019-12-07T19:33:50.649+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953801+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:50.649+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954030+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:50.649+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953989+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:50.649+0000 7f2b9171a700 20 osd.6 15 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:50.649+0000 7f2b9171a700 20 osd.6 15 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:50.649+0000 7f2b9171a700 20 osd.6 15 scrub_load_below_threshold loadavg per cpu 0.06125 < max 5 = yes 2019-12-07T19:33:50.649+0000 7f2b9171a700 20 osd.6 15 sched_scrub load_is_low=1 2019-12-07T19:33:50.649+0000 7f2b9171a700 20 osd.6 15 sched_scrub done 2019-12-07T19:33:50.649+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:50.649+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:50.649+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:50.650+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.1 172.21.15.131:0/37078 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.107717608s delta_ub 0.002773421s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c800 con 0x5557e7518400 2019-12-07T19:33:50.650+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002146623s,0.002773421s]) 2019-12-07T19:33:50.650+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.105570985s delta_ub 18446744073.707404993s) v5 -- 0x5557e758c000 con 0x5557e7518400 2019-12-07T19:33:50.651+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.1 172.21.15.131:0/37078 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.107717608s delta_ub 0.002773421s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e74cdc00 2019-12-07T19:33:50.651+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001957702s,0.002773421s]) 2019-12-07T19:33:50.651+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.105759906s delta_ub 18446744073.707593914s) v5 -- 0x5557e758c000 con 0x5557e74cdc00 2019-12-07T19:33:50.677+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.727+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.777+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.827+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.877+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.927+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.978+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.028+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.078+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.128+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.153+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.0 172.21.15.131:0/37079 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.000372524s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c800 con 0x5557e7518800 2019-12-07T19:33:51.153+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709425021s,0.000372524s]) 2019-12-07T19:33:51.153+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607818995s delta_ub 0.000126595s) v5 -- 0x5557e758c000 con 0x5557e7518800 2019-12-07T19:33:51.153+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.0 172.21.15.131:0/37079 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.000372524s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7518000 2019-12-07T19:33:51.153+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709243542s,0.000372524s]) 2019-12-07T19:33:51.153+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.608000474s delta_ub 0.000308074s) v5 -- 0x5557e758c000 con 0x5557e7518000 2019-12-07T19:33:51.178+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.228+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.236+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.2 172.21.15.131:0/37077 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.708091194s delta_ub 0.017361225s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c800 con 0x5557e74cd400 2019-12-07T19:33:51.236+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.2 172.21.15.131:0/37077 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.708091194s delta_ub 0.017361225s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e74ccc00 2019-12-07T19:33:51.236+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017056158s,0.017361225s]) 2019-12-07T19:33:51.236+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.691035036s delta_ub 18446744073.692495458s) v5 -- 0x5557e758c000 con 0x5557e74cd400 2019-12-07T19:33:51.236+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017036866s,0.017361225s]) 2019-12-07T19:33:51.236+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.691054328s delta_ub 18446744073.692514750s) v5 -- 0x5557e62b8c00 con 0x5557e74ccc00 2019-12-07T19:33:51.278+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.328+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.378+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.429+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.479+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.490+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:51.490+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:51.490+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:51.490+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:51.529+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:51.579+0000 7f2b8ad07700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258772 2019-12-07T19:33:51.579+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:51.579+0000 7f2b8ad07700 5 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.627+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:51.627+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:51.627+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954007+0000 last_rx_front 2019-12-07T19:33:45.953936+0000 2019-12-07T19:33:51.627+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953973+0000 last_rx_front 2019-12-07T19:33:45.953915+0000 2019-12-07T19:33:51.627+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953897+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:51.627+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:45.953925+0000 2019-12-07T19:33:51.627+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953801+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:51.627+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.954030+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:51.627+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:45.953545+0000 last_rx_back 2019-12-07T19:33:45.953989+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:51.627+0000 7f2b9171a700 20 osd.6 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:51.627+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:51.627+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:51.627+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:51.629+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.679+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.729+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.778+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.828+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.852+0000 7f2b724d6700 5 osd.6 15 heartbeat osd_stat(store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,7] op hist []) 2019-12-07T19:33:51.852+0000 7f2b724d6700 20 osd.6 15 check_full_status cur ratio 1.56826e-06, physical ratio 1.56826e-06, new state none 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.000308074s) v5 -- 0x5557e74f1c00 con 0x5557e7a49000 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.000308074s) v5 -- 0x5557e74f1800 con 0x5557e7a49400 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 18446744073.707593914s) v5 -- 0x5557e74f1a00 con 0x5557e7af0800 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 18446744073.707593914s) v5 -- 0x5557e758de00 con 0x5557e7af0c00 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 18446744073.692514750s) v5 -- 0x5557e75a0000 con 0x5557e7af1000 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 18446744073.692514750s) v5 -- 0x5557e75a0200 con 0x5557e7af1400 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 18446744073.706864486s) v5 -- 0x5557e75a0400 con 0x5557e7af1800 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 18446744073.706864486s) v5 -- 0x5557e75a0600 con 0x5557e7af1c00 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.005517570s) v5 -- 0x5557e75a0800 con 0x5557e748e000 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.005517570s) v5 -- 0x5557e75a0a00 con 0x5557e748e400 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 18446744073.697996725s) v5 -- 0x5557e75a0c00 con 0x5557e7a49800 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 18446744073.697996725s) v5 -- 0x5557e75a0e00 con 0x5557e7a49c00 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.019412398s) v5 -- 0x5557e75a1000 con 0x5557e7af0000 2019-12-07T19:33:51.852+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.019412398s) v5 -- 0x5557e75a1200 con 0x5557e7af0400 2019-12-07T19:33:51.852+0000 7f2b957b2700 0 -- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e748e000 msgr2=0x5557e7480100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:33:51.852+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6814/36938 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.303296475s delta_ub 18446744073.704459815s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f1a00 con 0x5557e748e000 2019-12-07T19:33:51.852+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.953801+0000 -> 2019-12-07T19:33:51.854155+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:51.852+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6830/36939 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.289554355s delta_ub 18446744073.690717695s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af0000 2019-12-07T19:33:51.852+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704186696s,18446744073.704459815s]) 2019-12-07T19:33:51.852+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6806/36936 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.320426262s delta_ub 0.012037986s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0200 con 0x5557e7a49800 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e748e000 msgr2=0x5557e7480100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 71 2019-12-07T19:33:51.853+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.954030+0000 -> 2019-12-07T19:33:51.854213+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e748e000 msgr2=0x5557e7480100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:51.853+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011706873s,0.012037986s]) 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e748e000 0x5557e7480100 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e748e000 0x5557e7480100 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:51.853+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.953989+0000 -> 2019-12-07T19:33:51.854236+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 2019-12-07T19:33:51.853+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690363833s,18446744073.690717695s]) 2019-12-07T19:33:51.853+0000 7f2b814f4700 20 osd.6 15 heartbeat_reset con 0x5557e748e000 s 0x5557e7aef040 2019-12-07T19:33:51.853+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6808/37077 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.325883057s delta_ub 0.017494781s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0200 con 0x5557e7af1000 2019-12-07T19:33:51.853+0000 7f2b814f4700 10 osd.6 15 heartbeat_reset failed hb con 0x5557e748e000 for osd.4, reopening 2019-12-07T19:33:51.853+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6827/37078 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.311094171s delta_ub 0.002705895s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af0c00 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6804/36936 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.320440200s delta_ub 0.012051924s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7a49c00 2019-12-07T19:33:51.853+0000 7f2b814f4700 1 -- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x5557e748e400 msgr2=0x5557e7480680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:51.853+0000 7f2b814f4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x5557e748e400 0x5557e7480680 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:51.853+0000 7f2b814f4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e75aa000 0x5557e75b0000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:51.853+0000 7f2b814f4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x5557e75aa400 0x5557e75b0580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:51.853+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.953897+0000 -> 2019-12-07T19:33:51.854321+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 2019-12-07T19:33:51.853+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017055775s,0.017494781s]) 2019-12-07T19:33:51.853+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.953973+0000 last_rx_front 2019-12-07T19:33:45.953915+0000 -> 2019-12-07T19:33:51.854333+0000 2019-12-07T19:33:51.853+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002255526s,0.002705895s]) 2019-12-07T19:33:51.853+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6814/37076 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.311695198s delta_ub 0.003306922s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0200 con 0x5557e7af1c00 2019-12-07T19:33:51.853+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854213+0000 last_rx_front 2019-12-07T19:33:45.953842+0000 -> 2019-12-07T19:33:51.854344+0000 2019-12-07T19:33:51.853+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:51.853882+0000) and older pending ping(s) 2019-12-07T19:33:51.853+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6816/37076 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.311725502s delta_ub 0.003337226s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af1800 2019-12-07T19:33:51.853+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011590145s,0.012051924s]) 2019-12-07T19:33:51.853+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 last_rx_front 2019-12-07T19:33:45.953925+0000 -> 2019-12-07T19:33:51.854362+0000 2019-12-07T19:33:51.853+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002827406s,0.003306922s]) 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6806/37077 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.325907057s delta_ub 0.017518781s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7af1400 2019-12-07T19:33:51.853+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.953866+0000 -> 2019-12-07T19:33:51.854372+0000 last_rx_front 2019-12-07T19:33:51.854362+0000 2019-12-07T19:33:51.853+0000 7f2b967b4700 25 osd.6 15 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:51.853882+0000) and older pending ping(s) 2019-12-07T19:33:51.853+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6828/36939 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.289592851s delta_ub 18446744073.690756191s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0200 con 0x5557e7af0400 2019-12-07T19:33:51.853+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002847979s,0.003337226s]) 2019-12-07T19:33:51.853+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854321+0000 last_rx_front 2019-12-07T19:33:45.953881+0000 -> 2019-12-07T19:33:51.854384+0000 2019-12-07T19:33:51.853+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:51.853882+0000) and older pending ping(s) 2019-12-07T19:33:51.853+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017016793s,0.017518781s]) 2019-12-07T19:33:51.853+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854236+0000 last_rx_front 2019-12-07T19:33:45.953963+0000 -> 2019-12-07T19:33:51.854399+0000 2019-12-07T19:33:51.853+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:51.853882+0000) and older pending ping(s) 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6826/37079 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308854641s delta_ub 0.000466365s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7a49400 2019-12-07T19:33:51.853+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690240092s,18446744073.690756191s]) 2019-12-07T19:33:51.853+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.954007+0000 last_rx_front 2019-12-07T19:33:45.953936+0000 -> 2019-12-07T19:33:51.854420+0000 2019-12-07T19:33:51.853+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709480501s,0.000466365s]) 2019-12-07T19:33:51.853+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6830/37079 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308899540s delta_ub 0.000511264s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0200 con 0x5557e7a49000 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6831/37078 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.311144812s delta_ub 0.002756536s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7af0800 2019-12-07T19:33:51.853+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.954007+0000 -> 2019-12-07T19:33:51.854436+0000 last_rx_front 2019-12-07T19:33:51.854420+0000 2019-12-07T19:33:51.853+0000 7f2b95fb3700 25 osd.6 15 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:51.853882+0000) and older pending ping(s) 2019-12-07T19:33:51.853+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709508823s,0.000511264s]) 2019-12-07T19:33:51.853+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:45.953973+0000 -> 2019-12-07T19:33:51.854454+0000 last_rx_front 2019-12-07T19:33:51.854333+0000 2019-12-07T19:33:51.853+0000 7f2b957b2700 25 osd.6 15 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:51.853882+0000) and older pending ping(s) 2019-12-07T19:33:51.853+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002184935s,0.002756536s]) 2019-12-07T19:33:51.853+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x5557e75aa400 0x5557e75b0580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e75aa000 0x5557e75b0000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:51.853+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x5557e75aa000 0x5557e75b0000 crc :-1 s=READY pgs=8 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:51.853+0000 7f2b967b4700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x5557e75aa400 0x5557e75b0580 crc :-1 s=READY pgs=8 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:51.878+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.929+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.979+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.029+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.079+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.129+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.179+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.229+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.279+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.329+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.335+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.2 172.21.15.131:0/37077 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.808438677s delta_ub 0.017518781s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0400 con 0x5557e74cd400 2019-12-07T19:33:52.335+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.2 172.21.15.131:0/37077 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.808438677s delta_ub 0.017518781s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f1a00 con 0x5557e74ccc00 2019-12-07T19:33:52.335+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016999748s,0.017518781s]) 2019-12-07T19:33:52.335+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.791438929s delta_ub 18446744073.692551868s) v5 -- 0x5557e75a1000 con 0x5557e74cd400 2019-12-07T19:33:52.335+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016979026s,0.017518781s]) 2019-12-07T19:33:52.335+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.791459651s delta_ub 18446744073.692572590s) v5 -- 0x5557e74f1800 con 0x5557e74ccc00 2019-12-07T19:33:52.380+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.430+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.480+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.507+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:52.507+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:52.507+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:52.507+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:52.530+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.580+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.630+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.659+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:52.659+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:52.659+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854436+0000 last_rx_front 2019-12-07T19:33:51.854420+0000 2019-12-07T19:33:52.659+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854454+0000 last_rx_front 2019-12-07T19:33:51.854333+0000 2019-12-07T19:33:52.659+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854321+0000 last_rx_front 2019-12-07T19:33:51.854384+0000 2019-12-07T19:33:52.659+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854372+0000 last_rx_front 2019-12-07T19:33:51.854362+0000 2019-12-07T19:33:52.659+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854155+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:52.659+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854213+0000 last_rx_front 2019-12-07T19:33:51.854344+0000 2019-12-07T19:33:52.659+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854236+0000 last_rx_front 2019-12-07T19:33:51.854399+0000 2019-12-07T19:33:52.659+0000 7f2b9171a700 20 osd.6 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:52.659+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:52.659+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:52.659+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:52.680+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.730+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.750+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.209356665s delta_ub 0.003337226s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e79f4800 2019-12-07T19:33:52.750+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.209356665s delta_ub 0.003337226s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e79f4c00 2019-12-07T19:33:52.750+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002796136s,0.003337226s]) 2019-12-07T19:33:52.750+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.206560529s delta_ub 18446744073.706755480s) v5 -- 0x5557e75a1000 con 0x5557e79f4800 2019-12-07T19:33:52.750+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002767547s,0.003337226s]) 2019-12-07T19:33:52.750+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.206589118s delta_ub 18446744073.706784069s) v5 -- 0x5557e74f1800 con 0x5557e79f4c00 2019-12-07T19:33:52.781+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.831+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.841+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.5 172.21.15.109:0/36936 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.308647276s delta_ub 0.012051924s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0200 con 0x5557e62f4c00 2019-12-07T19:33:52.841+0000 7f2b95fb3700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011570864s,0.012051924s]) 2019-12-07T19:33:52.841+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.297076412s delta_ub 18446744073.697980752s) v5 -- 0x5557e75a0600 con 0x5557e62f4c00 2019-12-07T19:33:52.841+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.5 172.21.15.109:0/36936 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.308647276s delta_ub 0.012051924s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0400 con 0x5557e7518c00 2019-12-07T19:33:52.841+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011503775s,0.012051924s]) 2019-12-07T19:33:52.841+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.297143501s delta_ub 18446744073.698047841s) v5 -- 0x5557e75a1000 con 0x5557e7518c00 2019-12-07T19:33:52.878+0000 7f2b937ae700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 6 (PID: 36932) UID: 0 2019-12-07T19:33:52.881+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.931+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.981+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.031+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.080+0000 7f2b937ae700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 6 (PID: 36932) UID: 0 2019-12-07T19:33:53.081+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.131+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.182+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.232+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.282+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.332+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.358+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e748e400 0x5557e7480680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:53.358+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e748e400 0x5557e7480680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:53.358+0000 7f2b957b2700 10 osd.6 15 ms_handle_authentication new session 0x5557e7aefba0 con 0x5557e748e400 entity client.admin addr 2019-12-07T19:33:53.358+0000 7f2b957b2700 10 osd.6 15 ms_handle_authentication session 0x5557e7aefba0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:53.358+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/481722811 conn(0x5557e748e400 0x5557e7480680 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).ready entity=client.4295 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:53.358+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4295 172.21.15.131:0/481722811 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x5557e75b4000 con 0x5557e748e400 2019-12-07T19:33:53.358+0000 7f2b88017700 20 osd.6 15 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:53.358+0000 7f2b88017700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:53.358+0000 7f2b88017700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:53.358+0000 7f2b88017700 20 osd.6 15 _dispatch 0x5557e75b4000 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:53.358+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/481722811 -- command_reply(tid 13: 0 ) v1 -- 0x5557e627ee00 con 0x5557e748e400 2019-12-07T19:33:53.381+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4295 172.21.15.131:0/481722811 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x5557e75b41e0 con 0x5557e748e400 2019-12-07T19:33:53.381+0000 7f2b88017700 20 osd.6 15 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:53.381+0000 7f2b88017700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:53.381+0000 7f2b88017700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:53.381+0000 7f2b88017700 20 osd.6 15 _dispatch 0x5557e75b41e0 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:53.382+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.383+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/481722811 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x5557e627e380 con 0x5557e748e400 2019-12-07T19:33:53.413+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/481722811 conn(0x5557e748e400 msgr2=0x5557e7480680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:53.413+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/481722811 conn(0x5557e748e400 msgr2=0x5557e7480680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:53.413+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/481722811 conn(0x5557e748e400 0x5557e7480680 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:53.413+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/481722811 conn(0x5557e748e400 0x5557e7480680 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:53.413+0000 7f2b88017700 2 osd.6 15 ms_handle_reset con 0x5557e748e400 session 0x5557e7aefba0 2019-12-07T19:33:53.432+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.482+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.532+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.548+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:53.548+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:53.548+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:53.548+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:53.582+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.629+0000 7f2b9171a700 10 osd.6 15 tick_without_osd_lock 2019-12-07T19:33:53.629+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:53.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854436+0000 last_rx_front 2019-12-07T19:33:51.854420+0000 2019-12-07T19:33:53.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854454+0000 last_rx_front 2019-12-07T19:33:51.854333+0000 2019-12-07T19:33:53.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854321+0000 last_rx_front 2019-12-07T19:33:51.854384+0000 2019-12-07T19:33:53.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854372+0000 last_rx_front 2019-12-07T19:33:51.854362+0000 2019-12-07T19:33:53.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854155+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:53.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854213+0000 last_rx_front 2019-12-07T19:33:51.854344+0000 2019-12-07T19:33:53.629+0000 7f2b9171a700 25 osd.6 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854236+0000 last_rx_front 2019-12-07T19:33:51.854399+0000 2019-12-07T19:33:53.629+0000 7f2b9171a700 20 osd.6 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:53.629+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:53.629+0000 7f2b9171a700 20 osd.6 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:53.629+0000 7f2b9171a700 10 osd.6 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:53.633+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.683+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.733+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.783+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.833+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.883+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.933+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.983+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.034+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.084+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.134+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.184+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.234+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.284+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.334+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.384+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.434+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.485+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.534+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.542+0000 7f2b92fad700 10 osd.6 15 tick 2019-12-07T19:33:54.542+0000 7f2b92fad700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:54.542+0000 7f2b92fad700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:54.542+0000 7f2b92fad700 20 osd.6 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:54.585+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:54.585+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.635+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.635+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.2 172.21.15.131:0/37077 5 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.108739871s delta_ub 0.017484426s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f1a00 con 0x5557e74ccc00 2019-12-07T19:33:54.636+0000 7f2b957b2700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017039995s,0.017484426s]) 2019-12-07T19:33:54.636+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.091699876s delta_ub 18446744073.692511621s) v5 -- 0x5557e7a9b600 con 0x5557e74ccc00 2019-12-07T19:33:54.636+0000 7f2b957b2700 10 osd.6 15 maybe_share_map con 0x5557e7519800 v2:172.21.15.131:6804/37077 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.636+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.2 172.21.15.131:0/37077 5 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.108739871s delta_ub 0.017484426s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e74cd400 2019-12-07T19:33:54.636+0000 7f2b967b4700 20 osd.6 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016505291s,0.017484426s]) 2019-12-07T19:33:54.636+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.092234580s delta_ub 18446744073.693046325s) v5 -- 0x5557e75a1000 con 0x5557e74cd400 2019-12-07T19:33:54.637+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 1 ==== osd_map(16..17 src has 1..17) v4 ==== 1760+0+0 (crc 0 0 0) 0x5557e74cafc0 con 0x5557e7519800 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 osd.6 15 OSD::ms_dispatch: osd_map(16..17 src has 1..17) v4 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 osd.6 15 do_waiters -- start 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 osd.6 15 do_waiters -- finish 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 osd.6 15 _dispatch 0x5557e74cafc0 osd_map(16..17 src has 1..17) v4 2019-12-07T19:33:54.637+0000 7f2b834f8700 3 osd.6 15 handle_osd_map epochs [16,17], i have 15, src has [1,17] 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 osd.6 15 handle_osd_map got inc map for epoch 16 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3630e826:::osdmap.15:0# 0x0~0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:54.637+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 msgr2=0x5557e7527600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 90 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e74d91f0 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e74d8ee0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e74d9b90 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x5557e74d9c00 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:54.637+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 msgr2=0x5557e7527600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3630e826:::osdmap.15:0# 0x0~1168 = 4456 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 osd.6 15 add_map_bl 15 4456 bytes 2019-12-07T19:33:54.637+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 crc :-1 s=READY pgs=10 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:54.637+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 crc :-1 s=READY pgs=10 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 osd.6 15 got_full_map 16, nothing requested 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 osd.6 15 handle_osd_map got inc map for epoch 17 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3cb0e826:::osdmap.16:0# 0x0~0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3cb0e826:::osdmap.16:0# key 0x7f7fffffffffffffff3cb0e8'&!osdmap.16!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3cb0e826:::osdmap.16:0# 0x0~0 = -2 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 osd.6 15 got_full_map 17, nothing requested 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 osd.6 15 handle_osd_map pg_num_history pg_num_history(e17 pg_nums {} deleted_pools ) 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps purged_snaps {16={},17={}} 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 osd.6 15 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e17 [1,17] lci=[13,17]) 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7121800 seq 9 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c423972f:::inc_osdmap.16:0# key 0x7f7fffffffffffffffc42397'/!inc_osdmap.16!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1059 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~1b9 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xe47 on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121800 1 blobs 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x25000~1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x25000~1000] 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c60e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c61c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c60e0 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c61c0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c60e0) get_ref 0x0~1b9 Blob(0x5557e75c60e0 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c61c0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c60e0) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1b9: 0x0~1b9 Blob(0x5557e75c60e0 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1b9) SharedBlob(0x5557e75c61c0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e75c61d8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1b9 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556608) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 = 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3cb0e826:::osdmap.16:0# key 0x7f7fffffffffffffff3cb0e8'&!osdmap.16!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3cb0e826:::osdmap.16:0# 0x0~1235 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1060 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3cb0e826:::osdmap.16:0# 0x0~1235 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~235 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xdcb on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121800 2 blobs 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x26000~2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x26000~2000] 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c62a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c62a0 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6310 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c62a0) get_ref 0x0~1000 Blob(0x5557e75c62a0 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6310 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c62a0) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e75c62a0 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e75c6310 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e75c6328 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c63f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6380 blob([!~1000,0x27000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c63f0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c6380) get_ref 0x1000~235 Blob(0x5557e75c6380 blob([!~1000,0x27000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e75c63f0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c6380) get_ref init 0x2000, 1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~235: 0x1000~235 Blob(0x5557e75c6380 blob([!~1000,0x27000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,235]) SharedBlob(0x5557e75c63f0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e75c6408 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1235 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556888) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3cb0e826:::osdmap.16:0# 0x0~1235 = 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:cfa3972f:::inc_osdmap.17:0# key 0x7f7fffffffffffffffcfa397'/!inc_osdmap.17!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1061 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~4f3 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xb0d on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121800 1 blobs 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x28000~1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x28000~1000] 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c65b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6540 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c65b0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c6540) get_ref 0x0~4f3 Blob(0x5557e75c6540 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c65b0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c6540) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~4f3: 0x0~4f3 Blob(0x5557e75c6540 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4f3) SharedBlob(0x5557e75c65b0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e75c65c8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x4f3 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556b08) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 = 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3340e826:::osdmap.17:0# key 0x7f7fffffffffffffff3340e8'&!osdmap.17!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3340e826:::osdmap.17:0# 0x0~1292 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1062 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3340e826:::osdmap.17:0# 0x0~1292 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~292 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xd6e on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121800 2 blobs 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x29000~2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x29000~2000] 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6690 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6700 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c6690) get_ref 0x0~1000 Blob(0x5557e75c6690 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6700 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c6690) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e75c6690 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e75c6700 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e75c6718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c67e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6770 blob([!~1000,0x2a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c67e0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c6770) get_ref 0x1000~292 Blob(0x5557e75c6770 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e75c67e0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e75c6770) get_ref init 0x2000, 1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~292: 0x1000~292 Blob(0x5557e75c6770 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,292]) SharedBlob(0x5557e75c67e0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e75c67f8 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1292 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556d88) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3340e826:::osdmap.17:0# 0x0~1292 = 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~12 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121800 0 blobs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121800 0 blobs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:54.637+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121800 cost 40864 (7 ios * 4000 + 12864 bytes) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121800 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7556500,0x5557e7556780,0x5557e7556a00,0x5557e7556c80 shared_blobs 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556608) update #-1:c423972f:::inc_osdmap.16:0# 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556608) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c423972f:::inc_osdmap.16:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556888) update #-1:3cb0e826:::osdmap.16:0# 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556888) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3cb0e826:::osdmap.16:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556b08) update #-1:cfa3972f:::inc_osdmap.17:0# 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556b08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:cfa3972f:::inc_osdmap.17:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556d88) update #-1:3340e826:::osdmap.17:0# 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7556d88) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:54.637+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3340e826:::osdmap.17:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:54.638+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121800 allocated 0x[25000~6000] released 0x[] 2019-12-07T19:33:54.638+0000 7f2b834f8700 10 freelist allocate 0x25000~6000 2019-12-07T19:33:54.638+0000 7f2b834f8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:54.638+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 prepare 2019-12-07T19:33:54.638+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7121800 2019-12-07T19:33:54.638+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 aio_wait 2019-12-07T19:33:54.638+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121800 2019-12-07T19:33:54.638+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 io_done 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9014 < min_flush_size 524288 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7556500 had 1 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7556780 had 1 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7556a00 had 1 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7556c80 had 1 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9061 < min_flush_size 524288 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9061 < min_flush_size 524288 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0xa528~2365 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0xa528 0xa528~2365 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xa528 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x528 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x88d and padding block with 0x773 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0xc88d 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:54.638+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 1 in 0.000194043s (0.000047791s flush + 0.000146252s kv commit) 2019-12-07T19:33:54.638+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121800> 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <0x5557e79edb80> 2019-12-07T19:33:54.638+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 kv_submitted 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121800 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7121800 osr 0x5557e76ce500 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) prepare_write seq 7 0x1b000~1000 crc 3e33509d 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6 15 _committed_osd_maps 16..17 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6 15 advance to epoch 16 (<= last 17 <= newest_map 17) 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) prepare_write seq 7 0x2000~1000 crc 69714ebe 2019-12-07T19:33:54.638+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 deferred_cleanup 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92300 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e70c7900,0x5557e70c7b80 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6 16 advance to epoch 17 (<= last 17 <= newest_map 17) 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d90b8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab5f0 space 0x5557e74d90b8 0x1000~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a287f8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab4d0 space 0x5557e7a287f8 0x0~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a91f28 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab560 space 0x5557e7a91f28 0x0~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7f2b85cfd700 7 osd.6 17 consume_map version 17 2019-12-07T19:33:54.638+0000 7f2b85cfd700 20 osd.6 17 prune_pg_created 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6:0.consume_map 17 (was 15) 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92300 done 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6:1.consume_map 17 (was 15) 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 deferred_cleanup 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6:2.consume_map 17 (was 15) 2019-12-07T19:33:54.638+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92300 [] 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6:3.consume_map 17 (was 15) 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6:4.consume_map 17 (was 15) 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6:5.consume_map 17 (was 15) 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6:6.consume_map 17 (was 15) 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6:7.consume_map 17 (was 15) 2019-12-07T19:33:54.638+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:54.638+0000 7f2b85cfd700 10 osd.6 17 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:54.638+0000 7f2b85cfd700 7 osd.6 17 activate_map version 17 2019-12-07T19:33:54.638+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 deferred_cleanup 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7556000,0x5557e7556280 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d8ef8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab9e0 space 0x5557e74d8ef8 0x0~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d92e8 in 0x5557e6269340) _finish_write discard buffer(0x5557e6279dd0 space 0x5557e74d92e8 0x0~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7f2b7ace7700 20 osd.6 17 agent_entry empty queue 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d9c18 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aad80 space 0x5557e74d9c18 0x1000~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121800 deferred_queued 2019-12-07T19:33:54.638+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:54.638+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:54.638+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:54.669+0000 7f2b9171a700 10 osd.6 17 tick_without_osd_lock 2019-12-07T19:33:54.669+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffd5000/0x40000000/0x1680000000, data 0xc3f9/0x29000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:54.669+0000 7f2b9171a700 25 osd.6 17 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854436+0000 last_rx_front 2019-12-07T19:33:51.854420+0000 2019-12-07T19:33:54.669+0000 7f2b9171a700 25 osd.6 17 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854454+0000 last_rx_front 2019-12-07T19:33:51.854333+0000 2019-12-07T19:33:54.669+0000 7f2b9171a700 25 osd.6 17 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854321+0000 last_rx_front 2019-12-07T19:33:51.854384+0000 2019-12-07T19:33:54.669+0000 7f2b9171a700 25 osd.6 17 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854372+0000 last_rx_front 2019-12-07T19:33:51.854362+0000 2019-12-07T19:33:54.669+0000 7f2b9171a700 25 osd.6 17 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854155+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:54.669+0000 7f2b9171a700 25 osd.6 17 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854213+0000 last_rx_front 2019-12-07T19:33:51.854344+0000 2019-12-07T19:33:54.669+0000 7f2b9171a700 25 osd.6 17 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854236+0000 last_rx_front 2019-12-07T19:33:51.854399+0000 2019-12-07T19:33:54.669+0000 7f2b9171a700 20 osd.6 17 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:54.669+0000 7f2b9171a700 20 osd.6 17 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:54.669+0000 7f2b9171a700 20 osd.6 17 scrub_load_below_threshold loadavg per cpu 0.05625 < max 5 = yes 2019-12-07T19:33:54.669+0000 7f2b9171a700 20 osd.6 17 sched_scrub load_is_low=1 2019-12-07T19:33:54.669+0000 7f2b9171a700 20 osd.6 17 sched_scrub done 2019-12-07T19:33:54.669+0000 7f2b9171a700 10 osd.6 17 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:54.669+0000 7f2b9171a700 20 osd.6 17 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:54.669+0000 7f2b9171a700 10 osd.6 17 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:54.685+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.691+0000 7f2b937ae700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 6 (PID: 36932) UID: 0 2019-12-07T19:33:54.735+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.750+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.1 172.21.15.131:0/37078 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.208013836s delta_ub 0.002756536s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0400 con 0x5557e7518400 2019-12-07T19:33:54.750+0000 7f2b967b4700 20 osd.6 17 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002277789s,0.002756536s]) 2019-12-07T19:33:54.750+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.205736047s delta_ub 18446744073.707273827s) v5 -- 0x5557e75a1000 con 0x5557e7518400 2019-12-07T19:33:54.750+0000 7f2b967b4700 10 osd.6 17 maybe_share_map con 0x5557e79cac00 v2:172.21.15.131:6823/37078 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.750+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.1 172.21.15.131:0/37078 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.208013836s delta_ub 0.002756536s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e75a0400 con 0x5557e74cdc00 2019-12-07T19:33:54.750+0000 7f2b967b4700 20 osd.6 17 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002239158s,0.002756536s]) 2019-12-07T19:33:54.750+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.205774678s delta_ub 18446744073.707312458s) v5 -- 0x5557e75a0a00 con 0x5557e74cdc00 2019-12-07T19:33:54.771+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.7 172.21.15.109:0/36939 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.690756191s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f1800 con 0x5557e74cc800 2019-12-07T19:33:54.771+0000 7f2b957b2700 20 osd.6 17 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690300874s,18446744073.690756191s]) 2019-12-07T19:33:54.771+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.226899905s delta_ub 0.019250742s) v5 -- 0x5557e7a9b600 con 0x5557e74cc800 2019-12-07T19:33:54.771+0000 7f2b957b2700 10 osd.6 17 maybe_share_map con 0x5557e79f4000 v2:172.21.15.109:6826/36939 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.771+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.7 172.21.15.109:0/36939 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.690756191s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f1a00 con 0x5557e748f800 2019-12-07T19:33:54.771+0000 7f2b957b2700 20 osd.6 17 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690229011s,18446744073.690756191s]) 2019-12-07T19:33:54.771+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.226971768s delta_ub 0.019322605s) v5 -- 0x5557e7a9b600 con 0x5557e748f800 2019-12-07T19:33:54.785+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.792+0000 7f2b937ae700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 6 (PID: 36932) UID: 0 2019-12-07T19:33:54.835+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.885+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.935+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.985+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.035+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.085+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.136+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.158+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.4 172.21.15.109:0/36938 6 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.608678189s delta_ub 18446744073.704527917s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9a600 con 0x5557e74cc400 2019-12-07T19:33:55.158+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.4 172.21.15.109:0/36938 6 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.608678189s delta_ub 18446744073.704527917s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7519000 2019-12-07T19:33:55.158+0000 7f2b95fb3700 20 osd.6 17 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704100835s,18446744073.704527917s]) 2019-12-07T19:33:55.158+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.614128970s delta_ub 0.005450781s) v5 -- 0x5557e75a0600 con 0x5557e74cc400 2019-12-07T19:33:55.158+0000 7f2b95fb3700 10 osd.6 17 maybe_share_map con 0x5557e79cbc00 v2:172.21.15.109:6810/36938 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:55.158+0000 7f2b967b4700 20 osd.6 17 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704074211s,18446744073.704527917s]) 2019-12-07T19:33:55.158+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.614155594s delta_ub 0.005477405s) v5 -- 0x5557e75a0a00 con 0x5557e7519000 2019-12-07T19:33:55.186+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.236+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.286+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.336+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.386+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.436+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.486+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.520+0000 7f2b83cf9700 20 osd.6 17 reports for 0 queries 2019-12-07T19:33:55.520+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.6 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x5557e79c4700 con 0x5557e7a48000 2019-12-07T19:33:55.520+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x5557e79c5180 con 0x5557e7a48000 2019-12-07T19:33:55.537+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.578+0000 7f2b92fad700 10 osd.6 17 tick 2019-12-07T19:33:55.578+0000 7f2b92fad700 10 osd.6 17 do_waiters -- start 2019-12-07T19:33:55.578+0000 7f2b92fad700 10 osd.6 17 do_waiters -- finish 2019-12-07T19:33:55.578+0000 7f2b92fad700 20 osd.6 17 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:55.587+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 46412 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.609+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 16 ==== pg_create2(e18 {1.cs0=18,2019-12-07T19:33:55.603918+0000}) v2 ==== 207+0+0 (crc 0 0 0) 0x5557e74ca240 con 0x5557e6f87000 2019-12-07T19:33:55.609+0000 7f2b967b4700 7 osd.6 17 handle_fast_pg_create pg_create2(e18 {1.cs0=18,2019-12-07T19:33:55.603918+0000}) v2 from mon.0 2019-12-07T19:33:55.609+0000 7f2b967b4700 20 osd.6 17 handle_fast_pg_create 1.cs0 e18@2019-12-07T19:33:55.603918+0000 history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 pi ([0,0] all_participants= intervals=) 2019-12-07T19:33:55.609+0000 7f2b967b4700 15 osd.6 17 enqueue_peering_evt 1.cs0 epoch_sent: 18 epoch_requested: 18 NullEvt +create_info 2019-12-07T19:33:55.609+0000 7f2b967b4700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.609+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.609+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.609+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.609+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.609+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 map 18 > 17 2019-12-07T19:33:55.609+0000 7f2b784e2700 20 osd.6 op_wq _add_slot_waiter 1.cs0 peering, item epoch is 18, will wait on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.609+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.614+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79f5000 0x5557e7a96000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:55.614+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79f5000 0x5557e7a96000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:55.614+0000 7f2b957b2700 10 osd.6 17 ms_handle_authentication new session 0x5557e75589c0 con 0x5557e79f5000 entity osd.2 addr 2019-12-07T19:33:55.614+0000 7f2b957b2700 10 osd.6 17 ms_handle_authentication session 0x5557e75589c0 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 1 ==== pg_query2(1.6s1 1.6s1 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79ca800 2019-12-07T19:33:55.615+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.6s1 epoch_sent: 18 epoch_requested: 18 MQuery 1.6s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.615+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 2 ==== pg_query2(1.1s2 1.1s2 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79ca800 2019-12-07T19:33:55.615+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.1s2 epoch_sent: 18 epoch_requested: 18 MQuery 1.1s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.615+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 3 ==== pg_query2(1.fs5 1.fs5 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79ca800 2019-12-07T19:33:55.615+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.fs5 epoch_sent: 18 epoch_requested: 18 MQuery 1.fs5 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.615+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs5 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.615+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.615+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 map 18 > 17 2019-12-07T19:33:55.615+0000 7f2b79ce5700 20 osd.6 op_wq _add_slot_waiter 1.1s2 peering, item epoch is 18, will wait on OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e79f5000 0x5557e7a96000 crc :-1 s=SESSION_ACCEPTING pgs=14 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x5557e7519800 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e79f5000 0x5557e7a96000 crc :-1 s=SESSION_ACCEPTING pgs=14 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:55.615+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 4 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (crc 0 0 0) 0x5557e70c38c0 con 0x5557e79ca800 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 osd.6 17 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 do_waiters -- start 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 do_waiters -- finish 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 osd.6 17 _dispatch 0x5557e70c38c0 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7f2b834f8700 3 osd.6 17 handle_osd_map epochs [16,18], i have 17, src has [1,18] 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 handle_osd_map got inc map for epoch 18 2019-12-07T19:33:55.615+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3340e826:::osdmap.17:0# 0x0~0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1292 size 0x1292 (4754) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c6690 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e75c6700 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c6770 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,292]) SharedBlob(0x5557e75c67e0 sbid 0x0)) need 0x1000~292 cache has 0x[1000~292] 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3340e826:::osdmap.17:0# 0x0~1292 = 4754 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 add_map_bl 17 4754 bytes 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 osd.6 17 got_full_map 18, nothing requested 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 handle_osd_map recording new pool 1 pg_num 16 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 osd.6 17 handle_osd_map pg_num_history pg_num_history(e18 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=READY pgs=14 cs=1 l=0 rx=0 tx=0).ready entity=osd.2 client_cookie=18704efb08e1a046 server_cookie=2a616824987376d2 in_seq=1 out_seq=0 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps purged_snaps {18={}} 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e18 [1,18] lci=[13,18]) 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7121500 seq 10 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c15b972f:::inc_osdmap.18:0# key 0x7f7fffffffffffffffc15b97'/!inc_osdmap.18!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:55.615+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1063 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~259 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xda7 on front/back, now 0x0~1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 1 blobs 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2b000~1000/1000,1000,0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2b000~1000] 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d9420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d9420 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9f80 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d9420) get_ref 0x0~259 Blob(0x5557e74d9420 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9f80 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d9420) get_ref init 0x1000, 1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~259: 0x0~259 Blob(0x5557e74d9420 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x259) SharedBlob(0x5557e74d9f80 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e74d9f98 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x259 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557288) dirty_range mark inline shard dirty 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 = 0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:39c0e826:::osdmap.18:0# key 0x7f7fffffffffffffff39c0e8'&!osdmap.18!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:55.615+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1064 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:39c0e826:::osdmap.18:0# 0x0~13ff - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~3ff 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 2 ==== pg_query2(1.5s1 1.5s1 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x5557e7556f00 con 0x5557e7519800 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xc01 on front/back, now 0x0~1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 2 blobs 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2c000~2000/1000,2000,0 2019-12-07T19:33:55.615+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.5s1 epoch_sent: 18 epoch_requested: 18 MQuery 1.5s1 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2c000~2000] 2019-12-07T19:33:55.615+0000 7f2b957b2700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.5s1 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d93b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d93b0 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d93b0) get_ref 0x0~1000 Blob(0x5557e74d93b0 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d93b0) get_ref init 0x1000, 1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e74d93b0 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e74d8620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e74d8638 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:55.615+0000 7f2b957b2700 0 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 msgr2=0x5557e7527600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d9490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a298f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d9490 blob([!~1000,0x2d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a298f0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d9490) get_ref 0x1000~3ff Blob(0x5557e74d9490 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e7a298f0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e74d9490) get_ref init 0x2000, 1000 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 msgr2=0x5557e7527600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 90 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~3ff: 0x1000~3ff Blob(0x5557e74d9490 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,3ff]) SharedBlob(0x5557e7a298f0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e7a29908 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 msgr2=0x5557e7527600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x13ff 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557508) dirty_range mark inline shard dirty 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff = 0 2019-12-07T19:33:55.615+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=READY pgs=14 cs=1 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~26 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:55.615+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=READY pgs=14 cs=1 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x38) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 0 blobs 2019-12-07T19:33:55.615+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.5s1 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x26 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.615+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:33:55.615+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.5s1 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.615+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 map 18 > 17 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:55.615+0000 7f2b77ce1700 20 osd.6 op_wq _add_slot_waiter 1.5s1 peering, item epoch is 18, will wait on OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.5s1 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:55.615+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:55.615+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:55.615+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 0 blobs 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121500 cost 23201 (4 ios * 4000 + 7201 bytes) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121500 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7557180,0x5557e7557400 shared_blobs 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557288) update #-1:c15b972f:::inc_osdmap.18:0# 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c15b972f:::inc_osdmap.18:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557508) update #-1:39c0e826:::osdmap.18:0# 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557508) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:39c0e826:::osdmap.18:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121500 allocated 0x[2b000~3000] released 0x[] 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 freelist allocate 0x2b000~3000 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 prepare 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7121500 2019-12-07T19:33:55.615+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79f5400 0x5557e7a98c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:55.615+0000 7f2b834f8700 2 osd.6 17 ms_handle_reset con 0x5557e79f5000 session 0x5557e75589c0 2019-12-07T19:33:55.615+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 3 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (unknown 0 0 0) 0x5557e74cbd40 con 0x5557e7519800 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 osd.6 17 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 do_waiters -- start 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 do_waiters -- finish 2019-12-07T19:33:55.615+0000 7f2b834f8700 20 osd.6 17 _dispatch 0x5557e74cbd40 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7f2b834f8700 3 osd.6 17 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.615+0000 7f2b834f8700 10 osd.6 17 no new maps here, dropping 2019-12-07T19:33:55.615+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79f5400 0x5557e7a98c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:55.616+0000 7f2b95fb3700 10 osd.6 17 ms_handle_authentication new session 0x5557e7558d00 con 0x5557e79f5400 entity osd.2 addr 2019-12-07T19:33:55.616+0000 7f2b95fb3700 10 osd.6 17 ms_handle_authentication session 0x5557e7558d00 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:55.616+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e79f5400 0x5557e7a98c00 crc :-1 s=SESSION_ACCEPTING pgs=17 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x5557e7519800 2019-12-07T19:33:55.616+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e79f5400 0x5557e7a98c00 crc :-1 s=SESSION_ACCEPTING pgs=17 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:55.616+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs5 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs5 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 map 18 > 17 2019-12-07T19:33:55.616+0000 7f2b834f8700 2 osd.6 17 ms_handle_reset con 0x5557e79f5400 session 0x5557e7558d00 2019-12-07T19:33:55.616+0000 7f2b76cdf700 20 osd.6 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 18, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs5 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:55.616+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 map 18 > 17 2019-12-07T19:33:55.616+0000 7f2b774e0700 20 osd.6 op_wq _add_slot_waiter 1.6s1 peering, item epoch is 18, will wait on OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=READY pgs=17 cs=2 l=0 rx=0 tx=0).ready entity=osd.2 client_cookie=18704efb08e1a046 server_cookie=2a616824987376d2 in_seq=3 out_seq=0 2019-12-07T19:33:55.616+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:55.616+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 1 ==== pg_query2(1.es5 1.es5 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79cac00 2019-12-07T19:33:55.616+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.es5 epoch_sent: 18 epoch_requested: 18 MQuery 1.es5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.616+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 2 ==== pg_query2(1.9s5 1.9s5 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79cac00 2019-12-07T19:33:55.616+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.9s5 epoch_sent: 18 epoch_requested: 18 MQuery 1.9s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.616+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 map 18 > 17 2019-12-07T19:33:55.616+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7f2b734d8700 20 osd.6 op_wq _add_slot_waiter 1.es5 peering, item epoch is 18, will wait on OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:55.616+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 map 18 > 17 2019-12-07T19:33:55.616+0000 7f2b75cdd700 20 osd.6 op_wq _add_slot_waiter 1.9s5 peering, item epoch is 18, will wait on OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:55.616+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 3 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (crc 0 0 0) 0x5557e7ade000 con 0x5557e79cac00 2019-12-07T19:33:55.616+0000 7f2b834f8700 20 osd.6 17 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.616+0000 7f2b834f8700 10 osd.6 17 do_waiters -- start 2019-12-07T19:33:55.616+0000 7f2b834f8700 10 osd.6 17 do_waiters -- finish 2019-12-07T19:33:55.616+0000 7f2b834f8700 20 osd.6 17 _dispatch 0x5557e7ade000 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.616+0000 7f2b834f8700 3 osd.6 17 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.616+0000 7f2b834f8700 10 osd.6 17 no new maps here, dropping 2019-12-07T19:33:55.616+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 4 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (crc 0 0 0) 0x5557e7ade240 con 0x5557e79cac00 2019-12-07T19:33:55.616+0000 7f2b834f8700 20 osd.6 17 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.616+0000 7f2b834f8700 10 osd.6 17 do_waiters -- start 2019-12-07T19:33:55.616+0000 7f2b834f8700 10 osd.6 17 do_waiters -- finish 2019-12-07T19:33:55.616+0000 7f2b834f8700 20 osd.6 17 _dispatch 0x5557e7ade240 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.616+0000 7f2b834f8700 3 osd.6 17 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.616+0000 7f2b834f8700 10 osd.6 17 no new maps here, dropping 2019-12-07T19:33:55.618+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 1 ==== pg_query2(1.2s3 1.2s3 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7557b80 con 0x5557e79cb800 2019-12-07T19:33:55.618+0000 7f2b967b4700 15 osd.6 17 enqueue_peering_evt 1.2s3 epoch_sent: 18 epoch_requested: 18 MQuery 1.2s3 from 0(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.618+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.2s3 from 0(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.2s3 from 0(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.618+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.2s3 from 0(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.618+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 map 18 > 17 2019-12-07T19:33:55.618+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 2 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (crc 0 0 0) 0x5557e74ca240 con 0x5557e79cb800 2019-12-07T19:33:55.618+0000 7f2b794e4700 20 osd.6 op_wq _add_slot_waiter 1.2s3 peering, item epoch is 18, will wait on OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.2s3 from 0(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f2b834f8700 20 osd.6 17 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.618+0000 7f2b834f8700 10 osd.6 17 do_waiters -- start 2019-12-07T19:33:55.618+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.618+0000 7f2b834f8700 10 osd.6 17 do_waiters -- finish 2019-12-07T19:33:55.618+0000 7f2b834f8700 20 osd.6 17 _dispatch 0x5557e74ca240 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.618+0000 7f2b834f8700 3 osd.6 17 handle_osd_map epochs [16,18], i have 18, src has [1,18] 2019-12-07T19:33:55.618+0000 7f2b834f8700 10 osd.6 17 no new maps here, dropping 2019-12-07T19:33:55.618+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 5 ==== pg_query2(1.7s1 1.7s1 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79cac00 2019-12-07T19:33:55.618+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.7s1 epoch_sent: 18 epoch_requested: 18 MQuery 1.7s1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.618+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 6 ==== pg_query2(1.as1 1.as1 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79cac00 2019-12-07T19:33:55.618+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.618+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.as1 epoch_sent: 18 epoch_requested: 18 MQuery 1.as1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.618+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.618+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 map 18 > 17 2019-12-07T19:33:55.618+0000 7f2b72cd7700 20 osd.6 op_wq _add_slot_waiter 1.7s1 peering, item epoch is 18, will wait on OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:55.618+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 7 ==== pg_query2(1.8s5 1.8s5 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79cac00 2019-12-07T19:33:55.618+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.8s5 epoch_sent: 18 epoch_requested: 18 MQuery 1.8s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.618+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.8s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f2b957b2700 0 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 msgr2=0x5557e7529180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:55.618+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 msgr2=0x5557e7529180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 93 2019-12-07T19:33:55.618+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 msgr2=0x5557e7529180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:55.618+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 crc :-1 s=READY pgs=10 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:55.618+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 crc :-1 s=READY pgs=10 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:55.618+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.8s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.618+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.8s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.618+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 map 18 > 17 2019-12-07T19:33:55.618+0000 7f2b7a4e6700 20 osd.6 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 18, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.8s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.619+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 map 18 > 17 2019-12-07T19:33:55.619+0000 7f2b754dc700 20 osd.6 op_wq _add_slot_waiter 1.as1 peering, item epoch is 18, will wait on OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.619+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79f5800 0x5557e7a99180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:55.619+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79f5800 0x5557e7a99180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:55.619+0000 7f2b957b2700 10 osd.6 17 ms_handle_authentication new session 0x5557e7558b60 con 0x5557e79f5800 entity osd.1 addr 2019-12-07T19:33:55.619+0000 7f2b957b2700 10 osd.6 17 ms_handle_authentication session 0x5557e7558b60 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:55.619+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79f5800 0x5557e7a99180 crc :-1 s=SESSION_ACCEPTING pgs=17 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x5557e79cac00 2019-12-07T19:33:55.619+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79f5800 0x5557e7a99180 crc :-1 s=SESSION_ACCEPTING pgs=17 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:55.619+0000 7f2b834f8700 2 osd.6 17 ms_handle_reset con 0x5557e79f5800 session 0x5557e7558b60 2019-12-07T19:33:55.619+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] reap_dead start 2019-12-07T19:33:55.620+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 unknown :-1 s=READY pgs=17 cs=1 l=0 rx=0 tx=0).ready entity=osd.1 client_cookie=949c5c84788cefc8 server_cookie=3b65d3eec6e35050 in_seq=7 out_seq=0 2019-12-07T19:33:55.620+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 3 ==== pg_query2(1.3s5 1.3s5 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79cbc00 2019-12-07T19:33:55.620+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 2 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (crc 0 0 0) 0x5557e7ade480 con 0x5557e79cbc00 2019-12-07T19:33:55.620+0000 7f2b834f8700 20 osd.6 17 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.620+0000 7f2b834f8700 10 osd.6 17 do_waiters -- start 2019-12-07T19:33:55.620+0000 7f2b834f8700 10 osd.6 17 do_waiters -- finish 2019-12-07T19:33:55.620+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.3s5 epoch_sent: 18 epoch_requested: 18 MQuery 1.3s5 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.620+0000 7f2b834f8700 20 osd.6 17 _dispatch 0x5557e7ade480 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.620+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s5 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.620+0000 7f2b834f8700 3 osd.6 17 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.620+0000 7f2b834f8700 10 osd.6 17 no new maps here, dropping 2019-12-07T19:33:55.620+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.620+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s5 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.620+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.620+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s5 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.620+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 map 18 > 17 2019-12-07T19:33:55.620+0000 7f2b78ce3700 20 osd.6 op_wq _add_slot_waiter 1.3s5 peering, item epoch is 18, will wait on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s5 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.620+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:55.620+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 4 ==== pg_query2(1.bs1 1.bs1 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (crc 0 0 0) 0x5557e7556f00 con 0x5557e79cbc00 2019-12-07T19:33:55.620+0000 7f2b957b2700 15 osd.6 17 enqueue_peering_evt 1.bs1 epoch_sent: 18 epoch_requested: 18 MQuery 1.bs1 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.620+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs1 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.620+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.620+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs1 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.620+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.620+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs1 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.620+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 map 18 > 17 2019-12-07T19:33:55.620+0000 7f2b74cdb700 20 osd.6 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 18, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs1 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.620+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:55.623+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 aio_wait 2019-12-07T19:33:55.623+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121500 2019-12-07T19:33:55.623+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 io_done 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7557180 had 1 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7557400 had 1 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0xc88d~226c to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0xc88d 0xc88d~226c to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xc88d 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x88d 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xaf9 and padding block with 0x507 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0xeaf9 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:55.623+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000155581s (0.000053695s flush + 0.000101886s kv commit) 2019-12-07T19:33:55.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121500> 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121500 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7121500 osr 0x5557e76ce500 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) _discard drop 7 0x1b000~1000 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6 17 _committed_osd_maps 18..18 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) prepare_write seq 8 0x1b000~1000 crc 45ebe265 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6 17 advance to epoch 18 (<= last 18 <= newest_map 18) 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) _discard drop 7 0x2000~1000 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) prepare_write seq 8 0x2000~1000 crc a1d00408 2019-12-07T19:33:55.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:55.623+0000 7f2b85cfd700 0 osd.6 18 crush map has features 288514119978713088, adjusting msgr requires for clients 2019-12-07T19:33:55.623+0000 7f2b85cfd700 0 osd.6 18 crush map has features 288514119978713088 was 288514050185503233, adjusting msgr requires for mons 2019-12-07T19:33:55.623+0000 7f2b85cfd700 0 osd.6 18 crush map has features 3314933069571702784, adjusting msgr requires for osds 2019-12-07T19:33:55.623+0000 7f2b85cfd700 7 osd.6 18 consume_map version 18 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:0.identify_splits_and_merges identify_splits_and_merges slot 1.8s5 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:1.identify_splits_and_merges identify_splits_and_merges slot 1.9s5 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:1.identify_splits_and_merges identify_splits_and_merges slot 1.1s2 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:2.identify_splits_and_merges identify_splits_and_merges slot 1.as1 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:2.identify_splits_and_merges identify_splits_and_merges slot 1.2s3 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs1 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:3.identify_splits_and_merges identify_splits_and_merges slot 1.3s5 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:4.identify_splits_and_merges identify_splits_and_merges slot 1.cs0 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:5.identify_splits_and_merges identify_splits_and_merges slot 1.5s1 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:6.identify_splits_and_merges identify_splits_and_merges slot 1.es5 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:6.identify_splits_and_merges identify_splits_and_merges slot 1.6s1 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:7.identify_splits_and_merges identify_splits_and_merges slot 1.7s1 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:7.identify_splits_and_merges identify_splits_and_merges slot 1.fs5 has no pg and waiting_for_split 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6 18 prune_pg_created 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6:0.consume_map 18 (was 17) 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:0.consume_map consume_map 1.8s5 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:0.consume_map consume_map 1.8s5 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6:1.consume_map 18 (was 17) 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.9s5 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.9s5 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:1._wake_pg_slot _wake_pg_slot 1.9s5 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.1s2 2019-12-07T19:33:55.623+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.1s2 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.8s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:1._wake_pg_slot _wake_pg_slot 1.1s2 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.8s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6:2.consume_map 18 (was 17) 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.as1 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.as1 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:2._wake_pg_slot _wake_pg_slot 1.as1 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b764de700 10 osd.6 18 handle_pg_query_nopg 1.8s5 2019-12-07T19:33:55.623+0000 7f2b764de700 10 osd.6 18 pg 1.8s5 dne 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.2s3 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.2s3 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:2._wake_pg_slot _wake_pg_slot 1.2s3 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s2 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6:3.consume_map 18 (was 17) 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.bs1 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.bs1 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b764de700 10 osd.6 18 send_incremental_map 17 -> 18 to 0x5557e79cac00 v2:172.21.15.131:6823/37078 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 osd.6 18 handle_pg_query_nopg 1.1s2 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 osd.6 18 pg 1.1s2 dne 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.3s5 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.3s5 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~0 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6:4.consume_map 18 (was 17) 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:4.consume_map consume_map 1.cs0 2019-12-07T19:33:55.623+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~259 size 0x259 (601) 2019-12-07T19:33:55.623+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:4.consume_map consume_map 1.cs0 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 osd.6 18 send_incremental_map 15 -> 18 to 0x5557e79ca800 v2:172.21.15.131:6812/37076 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:4._wake_pg_slot _wake_pg_slot 1.cs0 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e74d9420 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x259) SharedBlob(0x5557e74d9f80 sbid 0x0)) need 0x0~259 cache has 0x[0~259] 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6:5.consume_map 18 (was 17) 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:5.consume_map consume_map 1.5s1 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:5.consume_map consume_map 1.5s1 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s5 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 = 601 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:5._wake_pg_slot _wake_pg_slot 1.5s1 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b764de700 10 osd.6 18 add_map_inc_bl 18 601 bytes 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.3s5 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6:6.consume_map 18 (was 17) 2019-12-07T19:33:55.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.es5 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.es5 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:6._wake_pg_slot _wake_pg_slot 1.es5 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b78ce3700 10 osd.6 18 handle_pg_query_nopg 1.3s5 2019-12-07T19:33:55.623+0000 7f2b78ce3700 10 osd.6 18 pg 1.3s5 dne 2019-12-07T19:33:55.623+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:c423972f:::inc_osdmap.16:0# 0x0~0 2019-12-07T19:33:55.623+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b764de700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(18..18 src has 1..18) v4 -- 0x5557e74cb200 con 0x5557e79cac00 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.6s1 2019-12-07T19:33:55.623+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.5s1 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.6s1 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1b9 size 0x1b9 (441) 2019-12-07T19:33:55.623+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:6._wake_pg_slot _wake_pg_slot 1.6s1 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.5s1 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b764de700 10 osd.6 18 maybe_share_map con 0x5557e79cac00 v2:172.21.15.131:6823/37078 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.623+0000 7f2b85cfd700 10 osd.6:7.consume_map 18 (was 17) 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.7s1 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c60e0 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1b9) SharedBlob(0x5557e75c61c0 sbid 0x0)) need 0x0~1b9 cache has 0x[0~1b9] 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.7s1 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b78ce3700 10 osd.6 18 send_incremental_map 17 -> 18 to 0x5557e79cbc00 v2:172.21.15.109:6810/36938 2019-12-07T19:33:55.623+0000 7f2b73cd9700 10 osd.6 18 handle_pg_query_nopg 1.5s1 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:7._wake_pg_slot _wake_pg_slot 1.7s1 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b73cd9700 10 osd.6 18 pg 1.5s1 dne 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.fs5 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.fs5 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 no pg, should create on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.623+0000 7f2b764de700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.8s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e2f00 con 0x5557e79cac00 2019-12-07T19:33:55.623+0000 7f2b85cfd700 20 osd.6:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 = 441 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 osd.6 18 add_map_inc_bl 16 441 bytes 2019-12-07T19:33:55.623+0000 7f2b73cd9700 10 osd.6 18 send_incremental_map 17 -> 18 to 0x5557e7519800 v2:172.21.15.131:6804/37077 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~0 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.6s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.2s3 from 0(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~4f3 size 0x4f3 (1267) 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.2s3 from 0(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b774e0700 10 osd.6 18 handle_pg_query_nopg 1.6s1 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c6540 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4f3) SharedBlob(0x5557e75c65b0 sbid 0x0)) need 0x0~4f3 cache has 0x[0~4f3] 2019-12-07T19:33:55.623+0000 7f2b774e0700 10 osd.6 18 pg 1.6s1 dne 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 = 1267 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 osd.6 18 add_map_inc_bl 17 1267 bytes 2019-12-07T19:33:55.623+0000 7f2b794e4700 10 osd.6 18 handle_pg_query_nopg 1.2s3 2019-12-07T19:33:55.623+0000 7f2b794e4700 10 osd.6 18 pg 1.2s3 dne 2019-12-07T19:33:55.623+0000 7f2b774e0700 10 osd.6 18 send_incremental_map 15 -> 18 to 0x5557e79ca800 v2:172.21.15.131:6812/37076 2019-12-07T19:33:55.623+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(16..18 src has 1..18) v4 -- 0x5557e7ade000 con 0x5557e79ca800 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 osd.6 18 maybe_share_map con 0x5557e79ca800 v2:172.21.15.131:6812/37076 map epoch 15 -> 18 (shared) 2019-12-07T19:33:55.623+0000 7f2b73cd9700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(18..18 src has 1..18) v4 -- 0x5557e7aded80 con 0x5557e7519800 2019-12-07T19:33:55.623+0000 7f2b794e4700 10 osd.6 18 send_incremental_map 15 -> 18 to 0x5557e79cb800 v2:172.21.15.131:6822/37079 2019-12-07T19:33:55.623+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:18 sent:18 1.1s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e4960 con 0x5557e79ca800 2019-12-07T19:33:55.623+0000 7f2b73cd9700 10 osd.6 18 maybe_share_map con 0x5557e7519800 v2:172.21.15.131:6804/37077 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.623+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(16..18 src has 1..18) v4 -- 0x5557e7adf200 con 0x5557e79ca800 2019-12-07T19:33:55.623+0000 7f2b744da700 10 osd.6 18 _make_pg 1.cs0 2019-12-07T19:33:55.623+0000 7f2b73cd9700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify((query:18 sent:18 1.5s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e4f00 con 0x5557e7519800 2019-12-07T19:33:55.623+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(16..18 src has 1..18) v4 -- 0x5557e7adf680 con 0x5557e79cb800 2019-12-07T19:33:55.623+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:18 sent:18 1.6s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e21e0 con 0x5557e79ca800 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 osd.6 18 handle_pg_query_nopg 1.9s5 2019-12-07T19:33:55.623+0000 7f2b79ce5700 10 osd.6 18 pg 1.9s5 dne 2019-12-07T19:33:55.623+0000 7f2b794e4700 10 osd.6 18 maybe_share_map con 0x5557e79cb800 v2:172.21.15.131:6822/37079 map epoch 15 -> 18 (shared) 2019-12-07T19:33:55.623+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:55.623+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_notify((query:18 sent:18 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e2960 con 0x5557e79cb800 2019-12-07T19:33:55.623+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.9s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e6281c20 con 0x5557e79cac00 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es5 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b774e0700 10 osd.6 18 handle_pg_query_nopg 1.es5 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b774e0700 10 osd.6 18 pg 1.es5 dne 2019-12-07T19:33:55.623+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(18..18 src has 1..18) v4 -- 0x5557e7ade900 con 0x5557e79cbc00 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.c(unlocked)] enter Initial 2019-12-07T19:33:55.623+0000 7f2b78ce3700 10 osd.6 18 maybe_share_map con 0x5557e79cbc00 v2:172.21.15.109:6810/36938 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b794e4700 10 osd.6 18 handle_pg_query_nopg 1.as1 2019-12-07T19:33:55.623+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:18 sent:18 1.3s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e54a0 con 0x5557e79cbc00 2019-12-07T19:33:55.623+0000 7f2b794e4700 10 osd.6 18 pg 1.as1 dne 2019-12-07T19:33:55.623+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.es5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e23c0 con 0x5557e79cac00 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs1 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs1 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b78ce3700 10 osd.6 18 handle_pg_query_nopg 1.bs1 2019-12-07T19:33:55.623+0000 7f2b78ce3700 10 osd.6 18 pg 1.bs1 dne 2019-12-07T19:33:55.623+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:55.623+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.as1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e2000 con 0x5557e79cac00 2019-12-07T19:33:55.623+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:18 sent:18 1.bs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75b4960 con 0x5557e79cbc00 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs5 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs5 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b76cdf700 10 osd.6 18 handle_pg_query_nopg 1.fs5 2019-12-07T19:33:55.623+0000 7f2b76cdf700 10 osd.6 18 pg 1.fs5 dne 2019-12-07T19:33:55.623+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:55.623+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.623+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:18 sent:18 1.fs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75e4000 con 0x5557e79ca800 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s1 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.623+0000 7f2b76cdf700 10 osd.6 18 handle_pg_query_nopg 1.7s1 2019-12-07T19:33:55.623+0000 7f2b76cdf700 10 osd.6 18 pg 1.7s1 dne 2019-12-07T19:33:55.623+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.7s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x5557e75b54a0 con 0x5557e79cac00 2019-12-07T19:33:55.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:55.624+0000 7f2b744da700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:55.624+0000 7f2b744da700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:55.624+0000 7f2b744da700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:55.624+0000 7f2b744da700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:55.624+0000 7f2b744da700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 pg_epoch: 18 pg[1.c(unlocked)] enter NotTrimming 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.cs0_head fresh osr 0x5557e76ce000 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 0 up [6,5,4,1,2,3] acting [6,5,4,1,2,3] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a91dc0,0x5557e7a28310,0x5557e7a91ea0,0x5557e7a283f0,0x5557e7a29110] 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 19.080138855s + 0s) 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:55.624+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:55.624+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.cs0_head options 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:55.625695+0000 0 0.000000 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] activate_map 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] update_heartbeat_peers -> 1,2,3,4,5,6 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 18 need_heartbeat_peer_update 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:55.625735+0000 1 0.000030 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Start 2019-12-07T19:33:55.624+0000 7f2b744da700 1 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] state: transitioning to Primary 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] exit Start 2019-12-07T19:33:55.625748+0000 0 0.000000 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] enter Started/Primary/Peering 2019-12-07T19:33:55.624+0000 7f2b744da700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetInfo 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] build_prior all_probe 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] build_prior final: probe 1(3),2(4),3(5),4(2),5(1),6(0) down blocked_by {} 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] up_thru 0 < same_since 18, must notify monitor 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: no prior_set down osds, clearing prior_readable_until_ub 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.1(3) 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.2(4) 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.3(5) 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.4(2) 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.5(1) 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:55.624+0000 7f2b744da700 15 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] publish_stats_to_osd 0:0 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:55.624+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:55.624+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.cs3 1.cs3 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x5557e7606000 con 0x5557e79cac00 2019-12-07T19:33:55.624+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_query2(1.cs4 1.cs4 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x5557e7606280 con 0x5557e7519800 2019-12-07T19:33:55.624+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.cs5 1.cs5 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x5557e7606500 con 0x5557e79ca800 2019-12-07T19:33:55.624+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.cs2 1.cs2 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x5557e7606780 con 0x5557e79cbc00 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 18 send_incremental_map 15 -> 18 to 0x5557e79ca000 v2:172.21.15.109:6802/36936 2019-12-07T19:33:55.624+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(16..18 src has 1..18) v4 -- 0x5557e7adfd40 con 0x5557e79ca000 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 18 maybe_share_map con 0x5557e79ca000 v2:172.21.15.109:6802/36936 map epoch 15 -> 18 (shared) 2019-12-07T19:33:55.624+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.cs1 1.cs1 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x5557e7606a00 con 0x5557e79ca000 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7a92c00 seq 1 2019-12-07T19:33:55.624+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.cs0_head bits 4 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.cs0_head bits 4 = 0 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:30000000::::head# key 0x8080000000000000013000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:55.624+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1065 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:55.624+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._infover' <- _infover 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:55.624+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:55.624+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes 0x5557e7606c80 shared_blobs 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore.extentmap(0x5557e7606d88) update 0#1:30000000::::head# 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore.extentmap(0x5557e7606d88) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:30000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[] released 0x[] 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:55.624+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:55.624+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 18 handle_pg_create_info new pg pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6:4._attach_pg 1.cs0 0x5557e6e89000 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6:4._wake_pg_slot _wake_pg_slot 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 18 identify_splits_and_merges 1.cs0 e18 to e18 pg_nums {18=16} 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6:4._prime_splits 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 NullEvt +create_info 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:55.624+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 18 queue_want_up_thru now 18 (was 0), currently 0 2019-12-07T19:33:55.624+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 8 ==== pg_notify((query:18 sent:18 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x5557e75e23c0 con 0x5557e79cac00 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 18 send_alive up_thru currently 0 want 18 2019-12-07T19:33:55.624+0000 7f2b744da700 10 osd.6 18 send_alive want 18 2019-12-07T19:33:55.624+0000 7f2b957b2700 7 osd.6 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.1 2019-12-07T19:33:55.624+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_alive(want up_thru 18 have 18) v1 -- 0x5557e75e4960 con 0x5557e6f87000 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:55.624+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.624+0000 7f2b85cfd700 15 osd.6 18 enqueue_peering_evt 1.cs0 epoch_sent: 18 epoch_requested: 18 NullEvt 2019-12-07T19:33:55.624+0000 7f2b957b2700 15 osd.6 18 enqueue_peering_evt 1.cs0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 1(3) notify: (query:18 sent:18 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:55.624+0000 7f2b85cfd700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) 2019-12-07T19:33:55.624+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 1(3) notify: (query:18 sent:18 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 maybe_update_heartbeat_peers updating 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0xeaf9~56d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0xeaf9 0xeaf9~56d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xeaf9 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xaf9 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x66 and padding block with 0xf9a 2019-12-07T19:33:55.624+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 4 ==== pg_notify((query:18 sent:18 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x5557e75e4f00 con 0x5557e7519800 2019-12-07T19:33:55.624+0000 7f2b95fb3700 7 osd.6 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.2 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 adding neighbor peer osd.0 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 adding neighbor peer osd.5 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 adding neighbor peer osd.7 2019-12-07T19:33:55.624+0000 7f2b95fb3700 15 osd.6 18 enqueue_peering_evt 1.cs0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 2(4) notify: (query:18 sent:18 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 adding random peer osd.1 2019-12-07T19:33:55.624+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 2(4) notify: (query:18 sent:18 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.624+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 5 ==== pg_notify((query:18 sent:18 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (crc 0 0 0) 0x5557e75e23c0 con 0x5557e79ca800 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 adding random peer osd.2 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 adding random peer osd.3 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 adding random peer osd.4 2019-12-07T19:33:55.624+0000 7f2b957b2700 7 osd.6 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.3 2019-12-07T19:33:55.624+0000 7f2b85cfd700 10 osd.6 18 maybe_update_heartbeat_peers 7 peers, extras 0,1,2,3,4,5,7 2019-12-07T19:33:55.624+0000 7f2b85cfd700 7 osd.6 18 activate_map version 18 2019-12-07T19:33:55.624+0000 7f2b957b2700 15 osd.6 18 enqueue_peering_evt 1.cs0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 3(5) notify: (query:18 sent:18 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.624+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 3(5) notify: (query:18 sent:18 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.624+0000 7f2b7ace7700 20 osd.6 18 agent_entry empty queue 2019-12-07T19:33:55.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) pg 0x5557e6e89000 2019-12-07T19:33:55.624+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 NullEvt 2019-12-07T19:33:55.624+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:55.624+0000 7f2b784e2700 10 osd.6 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.624+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 1 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (unknown 0 0 0) 0x5557e7ade6c0 con 0x5557e79ca000 2019-12-07T19:33:55.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 1(3) notify: (query:18 sent:18 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.624+0000 7f2b834f8700 20 osd.6 18 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.624+0000 7f2b834f8700 10 osd.6 18 do_waiters -- start 2019-12-07T19:33:55.624+0000 7f2b834f8700 10 osd.6 18 do_waiters -- finish 2019-12-07T19:33:55.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.624+0000 7f2b834f8700 20 osd.6 18 _dispatch 0x5557e7ade6c0 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 1(3) notify: (query:18 sent:18 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x5557e6e89000 2019-12-07T19:33:55.624+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 2 ==== pg_notify((query:18 sent:18 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x5557e75e23c0 con 0x5557e79ca000 2019-12-07T19:33:55.624+0000 7f2b957b2700 7 osd.6 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.5 2019-12-07T19:33:55.624+0000 7f2b834f8700 3 osd.6 18 handle_osd_map epochs [16,18], i have 18, src has [1,18] 2019-12-07T19:33:55.624+0000 7f2b834f8700 10 osd.6 18 no new maps here, dropping 2019-12-07T19:33:55.624+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 1(3) notify: (query:18 sent:18 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0xf066 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:55.624+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:55.624+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.1(3) 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000140022s (0.000000094s flush + 0.000139928s kv commit) 2019-12-07T19:33:55.624+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:55.624+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:55.624+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92c00> 2019-12-07T19:33:55.624+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.625+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:55.624+0000 7f2b957b2700 15 osd.6 18 enqueue_peering_evt 1.cs0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 5(1) notify: (query:18 sent:18 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:55.625+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 5(1) notify: (query:18 sent:18 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.625+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 0x5557e7606c80 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.625+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:55.625+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:13.769035+0000, must = 0 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 2(4) notify: (query:18 sent:18 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 1,2,3,4,5,6 unchanged 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 2(4) notify: (query:18 sent:18 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x5557e6e89000 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 2(4) notify: (query:18 sent:18 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.2(4) 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:24.343811+0000, must = 0 2019-12-07T19:33:55.625+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 5 ==== pg_notify((query:18 sent:18 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (crc 0 0 0) 0x5557e75e23c0 con 0x5557e79cbc00 2019-12-07T19:33:55.625+0000 7f2b957b2700 7 osd.6 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.4 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 1,2,3,4,5,6 unchanged 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 2 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.625+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 3(5) notify: (query:18 sent:18 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.625+0000 7f2b957b2700 15 osd.6 18 enqueue_peering_evt 1.cs0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 4(2) notify: (query:18 sent:18 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.625+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 4(2) notify: (query:18 sent:18 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 3(5) notify: (query:18 sent:18 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x5557e6e89000 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 5(1) notify: (query:18 sent:18 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 3(5) notify: (query:18 sent:18 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.3(5) 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:22.020316+0000, must = 0 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 1,2,3,4,5,6 unchanged 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 5(1) notify: (query:18 sent:18 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x5557e6e89000 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 5(1) notify: (query:18 sent:18 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.5(1) 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 4(2) notify: (query:18 sent:18 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:15.621045+0000, must = 0 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 1,2,3,4,5,6 unchanged 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.625+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:55.625+0000 7f2b744da700 10 osd.6 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 4(2) notify: (query:18 sent:18 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x5557e6e89000 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.cs0 from 4(2) notify: (query:18 sent:18 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.4(2) 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.cs0 register next scrub, scrub time 2019-12-07T19:35:16.655986+0000, must = 0 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 1,2,3,4,5,6 unchanged 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:33:55.625+0000 7f2b784e2700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetInfo 2019-12-07T19:33:55.626422+0000 7 0.000219 2019-12-07T19:33:55.625+0000 7f2b784e2700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetLog 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.1(3) 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.2(4) 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.3(5) 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.4(2) 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.5(1) 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.6(0) 1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] calc_acting prefer osd.6(0) because it is current primary 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] For position 0: selecting up[i]: 6(0) For position 1: selecting up[i]: 5(1) For position 2: selecting up[i]: 4(2) For position 3: selecting up[i]: 1(3) For position 4: selecting up[i]: 2(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_async_recovery_ec result want=[6,5,4,1,2,3] async_recovery= 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] acting_recovery_backfill is 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting want=[6,5,4,1,2,3] backfill_targets= async_recovery_targets= 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: leaving GetLog 2019-12-07T19:33:55.625+0000 7f2b784e2700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetLog 2019-12-07T19:33:55.626523+0000 0 0.000000 2019-12-07T19:33:55.625+0000 7f2b784e2700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetMissing 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: still need up_thru update before going active 2019-12-07T19:33:55.625+0000 7f2b784e2700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetMissing 2019-12-07T19:33:55.626540+0000 0 0.000000 2019-12-07T19:33:55.625+0000 7f2b784e2700 5 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7046f00 seq 2 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 4032 (1 ios * 4000 + 32 bytes) 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes shared_blobs 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:55.625+0000 7f2b784e2700 10 osd.6 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0xf066~2e to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0xf066 0xf066~2e to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xf066 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x66 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x94 and padding block with 0xf6c 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0xf094 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:55.625+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000102657s (0.000000102s flush + 0.000102555s kv commit) 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:55.625+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.625+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:55.625+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:55.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:55.625+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:55.625+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:55.625+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:55.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.637+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.673+0000 7f2b9171a700 10 osd.6 18 tick_without_osd_lock 2019-12-07T19:33:55.673+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffd2000/0x40000000/0x1680000000, data 0xda65/0x2c000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:55.673+0000 7f2b9171a700 25 osd.6 18 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854436+0000 last_rx_front 2019-12-07T19:33:51.854420+0000 2019-12-07T19:33:55.673+0000 7f2b9171a700 25 osd.6 18 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854454+0000 last_rx_front 2019-12-07T19:33:51.854333+0000 2019-12-07T19:33:55.673+0000 7f2b9171a700 25 osd.6 18 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854321+0000 last_rx_front 2019-12-07T19:33:51.854384+0000 2019-12-07T19:33:55.673+0000 7f2b9171a700 25 osd.6 18 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854372+0000 last_rx_front 2019-12-07T19:33:51.854362+0000 2019-12-07T19:33:55.673+0000 7f2b9171a700 25 osd.6 18 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854155+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:55.673+0000 7f2b9171a700 25 osd.6 18 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854213+0000 last_rx_front 2019-12-07T19:33:51.854344+0000 2019-12-07T19:33:55.673+0000 7f2b9171a700 25 osd.6 18 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:51.853882+0000 last_rx_back 2019-12-07T19:33:51.854236+0000 last_rx_front 2019-12-07T19:33:51.854399+0000 2019-12-07T19:33:55.673+0000 7f2b9171a700 20 osd.6 18 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:55.673+0000 7f2b9171a700 20 osd.6 18 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:55.673+0000 7f2b9171a700 20 osd.6 18 scrub_load_below_threshold loadavg per cpu 0.05625 < max 5 = yes 2019-12-07T19:33:55.673+0000 7f2b9171a700 20 osd.6 18 sched_scrub load_is_low=1 2019-12-07T19:33:55.673+0000 7f2b9171a700 10 osd.6 18 sched_scrub 1.cs0 scheduled at 2019-12-07T19:35:16.655986+0000 > 2019-12-07T19:33:55.674570+0000 2019-12-07T19:33:55.673+0000 7f2b9171a700 20 osd.6 18 sched_scrub done 2019-12-07T19:33:55.673+0000 7f2b9171a700 10 osd.6 18 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:55.673+0000 7f2b9171a700 20 osd.6 18 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:55.673+0000 7f2b9171a700 10 osd.6 18 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:55.687+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.737+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.787+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.837+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.887+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.891+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e79f5c00 0x5557e7a99700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:55.891+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e79f5c00 0x5557e7a99700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:55.891+0000 7f2b957b2700 10 osd.6 18 ms_handle_authentication new session 0x5557e7aefba0 con 0x5557e79f5c00 entity client.admin addr 2019-12-07T19:33:55.891+0000 7f2b957b2700 10 osd.6 18 ms_handle_authentication session 0x5557e7aefba0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:55.891+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/808367751 conn(0x5557e79f5c00 0x5557e7a99700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4528 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:55.891+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4528 172.21.15.131:0/808367751 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x5557e75e23c0 con 0x5557e79f5c00 2019-12-07T19:33:55.891+0000 7f2b88017700 20 osd.6 18 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:55.891+0000 7f2b88017700 10 osd.6 18 do_waiters -- start 2019-12-07T19:33:55.891+0000 7f2b88017700 10 osd.6 18 do_waiters -- finish 2019-12-07T19:33:55.891+0000 7f2b88017700 20 osd.6 18 _dispatch 0x5557e75e23c0 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:55.891+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/808367751 -- command_reply(tid 13: 0 ) v1 -- 0x5557e627f340 con 0x5557e79f5c00 2019-12-07T19:33:55.913+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4528 172.21.15.131:0/808367751 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x5557e75e2000 con 0x5557e79f5c00 2019-12-07T19:33:55.913+0000 7f2b88017700 20 osd.6 18 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:33:55.913+0000 7f2b88017700 10 osd.6 18 do_waiters -- start 2019-12-07T19:33:55.913+0000 7f2b88017700 10 osd.6 18 do_waiters -- finish 2019-12-07T19:33:55.913+0000 7f2b88017700 20 osd.6 18 _dispatch 0x5557e75e2000 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:33:55.914+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/808367751 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x5557e627fc00 con 0x5557e79f5c00 2019-12-07T19:33:55.937+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.943+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/808367751 conn(0x5557e79f5c00 msgr2=0x5557e7a99700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:55.943+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/808367751 conn(0x5557e79f5c00 msgr2=0x5557e7a99700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:55.943+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/808367751 conn(0x5557e79f5c00 0x5557e7a99700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:55.943+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/808367751 conn(0x5557e79f5c00 0x5557e7a99700 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:55.943+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] reap_dead start 2019-12-07T19:33:55.943+0000 7f2b88017700 2 osd.6 18 ms_handle_reset con 0x5557e79f5c00 session 0x5557e7aefba0 2019-12-07T19:33:55.952+0000 7f2b724d6700 5 osd.6 18 heartbeat osd_stat(store_statfs(0x163ffd2000/0x40000000/0x1680000000, data 0xda65/0x2c000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,7] op hist []) 2019-12-07T19:33:55.953+0000 7f2b724d6700 20 osd.6 18 check_full_status cur ratio 1.94973e-06, physical ratio 1.94973e-06, new state none 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.000042793s) v5 -- 0x5557e7616000 con 0x5557e7a49000 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.000042793s) v5 -- 0x5557e7616200 con 0x5557e7a49400 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 18446744073.707312458s) v5 -- 0x5557e7616400 con 0x5557e7af0800 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 18446744073.707312458s) v5 -- 0x5557e7616600 con 0x5557e7af0c00 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 18446744073.693046325s) v5 -- 0x5557e7616800 con 0x5557e7af1000 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 18446744073.693046325s) v5 -- 0x5557e7616a00 con 0x5557e7af1400 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 18446744073.706784069s) v5 -- 0x5557e7616c00 con 0x5557e7af1800 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 18446744073.706784069s) v5 -- 0x5557e7616e00 con 0x5557e7af1c00 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.005477405s) v5 -- 0x5557e7617000 con 0x5557e75aa000 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.005477405s) v5 -- 0x5557e7617200 con 0x5557e75aa400 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 18446744073.698047841s) v5 -- 0x5557e7617400 con 0x5557e7a49800 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 18446744073.698047841s) v5 -- 0x5557e7617600 con 0x5557e7a49c00 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.019322605s) v5 -- 0x5557e7617800 con 0x5557e7af0000 2019-12-07T19:33:55.953+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.019322605s) v5 -- 0x5557e7617a00 con 0x5557e7af0400 2019-12-07T19:33:55.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6831/37078 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.411406004s delta_ub 0.002657567s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7af0800 2019-12-07T19:33:55.953+0000 7f2b957b2700 25 osd.6 18 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:51.854454+0000 -> 2019-12-07T19:33:55.954509+0000 last_rx_front 2019-12-07T19:33:51.854333+0000 2019-12-07T19:33:55.953+0000 7f2b957b2700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002391395s,0.002657567s]) 2019-12-07T19:33:55.953+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6830/36939 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.389897725s delta_ub 18446744073.690700904s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af0000 2019-12-07T19:33:55.953+0000 7f2b967b4700 25 osd.6 18 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:51.854236+0000 -> 2019-12-07T19:33:55.954528+0000 last_rx_front 2019-12-07T19:33:51.854399+0000 2019-12-07T19:33:55.953+0000 7f2b967b4700 10 osd.6 18 maybe_share_map con 0x5557e79f4000 v2:172.21.15.109:6826/36939 map epoch 17 -> 18 (as per caller) 2019-12-07T19:33:55.953+0000 7f2b967b4700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690415527s,18446744073.690700904s]) 2019-12-07T19:33:55.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6808/37077 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.426218238s delta_ub 0.017469801s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7616400 con 0x5557e7af1000 2019-12-07T19:33:55.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6814/36938 1 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.403683427s delta_ub 18446744073.704486606s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e75aa000 2019-12-07T19:33:55.953+0000 7f2b95fb3700 25 osd.6 18 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:51.854321+0000 -> 2019-12-07T19:33:55.954547+0000 last_rx_front 2019-12-07T19:33:51.854384+0000 2019-12-07T19:33:55.953+0000 7f2b95fb3700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017165344s,0.017469801s]) 2019-12-07T19:33:55.953+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6827/37078 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.411460502s delta_ub 0.002712065s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af0c00 2019-12-07T19:33:55.953+0000 7f2b967b4700 25 osd.6 18 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954509+0000 last_rx_front 2019-12-07T19:33:51.854333+0000 -> 2019-12-07T19:33:55.954559+0000 2019-12-07T19:33:55.953+0000 7f2b967b4700 25 osd.6 18 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:55.954243+0000) and older pending ping(s) 2019-12-07T19:33:55.953+0000 7f2b967b4700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002396109s,0.002712065s]) 2019-12-07T19:33:55.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6814/37076 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.412026926s delta_ub 0.003278489s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7616400 con 0x5557e7af1c00 2019-12-07T19:33:55.953+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6812/36938 1 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.403705452s delta_ub 18446744073.704508631s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e75aa400 2019-12-07T19:33:55.953+0000 7f2b957b2700 25 osd.6 18 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:51.854155+0000 -> 2019-12-07T19:33:55.954574+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 2019-12-07T19:33:55.953+0000 7f2b957b2700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704154869s,18446744073.704486606s]) 2019-12-07T19:33:55.953+0000 7f2b95fb3700 25 osd.6 18 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:51.854372+0000 last_rx_front 2019-12-07T19:33:51.854362+0000 -> 2019-12-07T19:33:55.954592+0000 2019-12-07T19:33:55.953+0000 7f2b95fb3700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002929373s,0.003278489s]) 2019-12-07T19:33:55.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6806/37077 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.426249203s delta_ub 0.017500766s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7af1400 2019-12-07T19:33:55.953+0000 7f2b967b4700 25 osd.6 18 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954574+0000 last_rx_front 2019-12-07T19:33:45.953943+0000 -> 2019-12-07T19:33:55.954605+0000 2019-12-07T19:33:55.953+0000 7f2b967b4700 25 osd.6 18 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:55.954243+0000) and older pending ping(s) 2019-12-07T19:33:55.953+0000 7f2b967b4700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704146016s,18446744073.704508631s]) 2019-12-07T19:33:55.953+0000 7f2b957b2700 25 osd.6 18 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954547+0000 last_rx_front 2019-12-07T19:33:51.854384+0000 -> 2019-12-07T19:33:55.954615+0000 2019-12-07T19:33:55.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6830/37079 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.409180863s delta_ub 0.000432426s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7616400 con 0x5557e7a49000 2019-12-07T19:33:55.953+0000 7f2b957b2700 25 osd.6 18 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:55.954243+0000) and older pending ping(s) 2019-12-07T19:33:55.953+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6816/37076 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.412088757s delta_ub 0.003340320s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af1800 2019-12-07T19:33:55.953+0000 7f2b957b2700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017127956s,0.017500766s]) 2019-12-07T19:33:55.953+0000 7f2b95fb3700 25 osd.6 18 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:51.854436+0000 -> 2019-12-07T19:33:55.954630+0000 last_rx_front 2019-12-07T19:33:51.854420+0000 2019-12-07T19:33:55.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6826/37079 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.409221055s delta_ub 0.000472618s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7a49400 2019-12-07T19:33:55.953+0000 7f2b95fb3700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.000045248s,0.000432426s]) 2019-12-07T19:33:55.953+0000 7f2b967b4700 25 osd.6 18 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:51.854372+0000 -> 2019-12-07T19:33:55.954643+0000 last_rx_front 2019-12-07T19:33:55.954592+0000 2019-12-07T19:33:55.953+0000 7f2b967b4700 25 osd.6 18 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:55.954243+0000) and older pending ping(s) 2019-12-07T19:33:55.953+0000 7f2b967b4700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002940140s,0.003340320s]) 2019-12-07T19:33:55.953+0000 7f2b957b2700 25 osd.6 18 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954630+0000 last_rx_front 2019-12-07T19:33:51.854420+0000 -> 2019-12-07T19:33:55.954652+0000 2019-12-07T19:33:55.953+0000 7f2b957b2700 25 osd.6 18 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:55.954243+0000) and older pending ping(s) 2019-12-07T19:33:55.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6806/36936 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.420778303s delta_ub 0.012029866s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7616400 con 0x5557e7a49800 2019-12-07T19:33:55.953+0000 7f2b957b2700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.000063503s,0.000472618s]) 2019-12-07T19:33:55.953+0000 7f2b95fb3700 25 osd.6 18 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:51.854213+0000 -> 2019-12-07T19:33:55.954662+0000 last_rx_front 2019-12-07T19:33:51.854344+0000 2019-12-07T19:33:55.953+0000 7f2b95fb3700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011610062s,0.012029866s]) 2019-12-07T19:33:55.953+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6804/36936 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.420812530s delta_ub 0.012064093s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7a49c00 2019-12-07T19:33:55.953+0000 7f2b957b2700 25 osd.6 18 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954662+0000 last_rx_front 2019-12-07T19:33:51.854344+0000 -> 2019-12-07T19:33:55.954672+0000 2019-12-07T19:33:55.953+0000 7f2b957b2700 25 osd.6 18 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:55.954243+0000) and older pending ping(s) 2019-12-07T19:33:55.953+0000 7f2b957b2700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011634725s,0.012064093s]) 2019-12-07T19:33:55.953+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6828/36939 4 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.390154608s delta_ub 18446744073.690957787s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f1a00 con 0x5557e7af0400 2019-12-07T19:33:55.953+0000 7f2b95fb3700 25 osd.6 18 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954528+0000 last_rx_front 2019-12-07T19:33:51.854399+0000 -> 2019-12-07T19:33:55.954756+0000 2019-12-07T19:33:55.953+0000 7f2b95fb3700 25 osd.6 18 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:55.954243+0000) and older pending ping(s) 2019-12-07T19:33:55.953+0000 7f2b95fb3700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690444823s,18446744073.690957787s]) 2019-12-07T19:33:55.987+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.037+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.088+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.138+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.188+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.238+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.288+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.338+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.341+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.5 172.21.15.109:0/36936 6 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.808987388s delta_ub 0.012064093s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e74f1a00 con 0x5557e62f4c00 2019-12-07T19:33:56.341+0000 7f2b95fb3700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011645960s,0.012064093s]) 2019-12-07T19:33:56.341+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.797341428s delta_ub 18446744073.697905656s) v5 -- 0x5557e7617a00 con 0x5557e62f4c00 2019-12-07T19:33:56.341+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.5 172.21.15.109:0/36936 6 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.808987388s delta_ub 0.012064093s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7518c00 2019-12-07T19:33:56.341+0000 7f2b967b4700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011560184s,0.012064093s]) 2019-12-07T19:33:56.341+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.797427204s delta_ub 18446744073.697991432s) v5 -- 0x5557e7617800 con 0x5557e7518c00 2019-12-07T19:33:56.388+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.438+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.452+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.0 172.21.15.131:0/37079 5 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.000472618s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b400 con 0x5557e7518800 2019-12-07T19:33:56.452+0000 7f2b967b4700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709489162s,0.000472618s]) 2019-12-07T19:33:56.452+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908085322s delta_ub 0.000062454s) v5 -- 0x5557e7617800 con 0x5557e7518800 2019-12-07T19:33:56.452+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.0 172.21.15.131:0/37079 5 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.000472618s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b400 con 0x5557e7518000 2019-12-07T19:33:56.452+0000 7f2b967b4700 20 osd.6 18 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [18446744073.709452256s,0.000472618s]) 2019-12-07T19:33:56.452+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37079 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908122228s delta_ub 0.000099360s) v5 -- 0x5557e7a9ba00 con 0x5557e7518000 2019-12-07T19:33:56.489+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.538+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.588+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:56.588+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:56.589+0000 7f2b8ad07700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258776 2019-12-07T19:33:56.589+0000 7f2b8ad07700 5 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50794 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.613+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 17 ==== osd_map(18..19 src has 1..19) v4 ==== 1298+0+0 (crc 0 0 0) 0x5557e7adf680 con 0x5557e6f87000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 osd.6 18 OSD::ms_dispatch: osd_map(18..19 src has 1..19) v4 2019-12-07T19:33:56.613+0000 7f2b88017700 10 osd.6 18 do_waiters -- start 2019-12-07T19:33:56.613+0000 7f2b88017700 10 osd.6 18 do_waiters -- finish 2019-12-07T19:33:56.613+0000 7f2b88017700 20 osd.6 18 _dispatch 0x5557e7adf680 osd_map(18..19 src has 1..19) v4 2019-12-07T19:33:56.613+0000 7f2b88017700 3 osd.6 18 handle_osd_map epochs [18,19], i have 18, src has [1,19] 2019-12-07T19:33:56.613+0000 7f2b88017700 10 osd.6 18 handle_osd_map got inc map for epoch 19 2019-12-07T19:33:56.613+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:39c0e826:::osdmap.18:0# 0x0~0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~13ff size 0x13ff (5119) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e74d93b0 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e74d8620 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e74d9490 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,3ff]) SharedBlob(0x5557e7a298f0 sbid 0x0)) need 0x1000~3ff cache has 0x[1000~3ff] 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff = 5119 2019-12-07T19:33:56.613+0000 7f2b88017700 10 osd.6 18 add_map_bl 18 5119 bytes 2019-12-07T19:33:56.613+0000 7f2b88017700 20 osd.6 18 got_full_map 19, nothing requested 2019-12-07T19:33:56.613+0000 7f2b88017700 20 osd.6 18 handle_osd_map pg_num_history pg_num_history(e19 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:56.613+0000 7f2b88017700 10 snap_mapper.record_purged_snaps purged_snaps {19={}} 2019-12-07T19:33:56.613+0000 7f2b88017700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:56.613+0000 7f2b88017700 10 osd.6 18 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e19 [1,19] lci=[13,19]) 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7a92600 seq 11 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:ca5b972f:::inc_osdmap.19:0# key 0x7f7fffffffffffffffca5b97'/!inc_osdmap.19!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:56.613+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1066 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~285 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xd7b on front/back, now 0x0~1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92600 1 blobs 2019-12-07T19:33:56.613+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:56.613+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2e000~1000/1000,1000,0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2e000~1000] 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d98f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d98f0 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28a10 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e74d98f0) get_ref 0x0~285 Blob(0x5557e74d98f0 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28a10 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e74d98f0) get_ref init 0x1000, 1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~285: 0x0~285 Blob(0x5557e74d98f0 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x5557e7a28a10 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 6 ==== PGlog(1.bs1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x5557e7a0e000 con 0x5557e79cbc00 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a28a28 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x285 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7606888) dirty_range mark inline shard dirty 2019-12-07T19:33:56.613+0000 7f2b957b2700 15 osd.6 18 enqueue_peering_evt 1.bs1 epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info 2019-12-07T19:33:56.613+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 = 0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:34c0e826:::osdmap.19:0# key 0x7f7fffffffffffffff34c0e8'&!osdmap.19!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:56.613+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.613+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.613+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:34c0e826:::osdmap.19:0# 0x0~140c 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1067 2019-12-07T19:33:56.613+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.613+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:34c0e826:::osdmap.19:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:56.613+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 map 19 > 18 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.613+0000 7f2b74cdb700 20 osd.6 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 19, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~40c 2019-12-07T19:33:56.613+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92600 2 blobs 2019-12-07T19:33:56.613+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:56.613+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2f000~2000/1000,2000,0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2f000~2000] 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d9ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d97a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d9ab0 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d97a0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e74d9ab0) get_ref 0x0~1000 Blob(0x5557e74d9ab0 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d97a0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e74d9ab0) get_ref init 0x1000, 1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e74d9ab0 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e74d97a0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e74d97b8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d9810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d9810 blob([!~1000,0x30000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8c40 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e74d9810) get_ref 0x1000~40c Blob(0x5557e74d9810 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8c40 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e74d9810) get_ref init 0x2000, 1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x5557e74d9810 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x5557e74d8c40 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e74d8c58 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x140c 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7606108) dirty_range mark inline shard dirty 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:34c0e826:::osdmap.19:0# 0x0~140c = 0 2019-12-07T19:33:56.613+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~26 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92600 0 blobs 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:56.613+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:56.613+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:56.613+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92600 0 blobs 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92600 cost 23258 (4 ios * 4000 + 7258 bytes) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92600 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7606000,0x5557e7606780 shared_blobs 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7606108) update #-1:34c0e826:::osdmap.19:0# 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7606108) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:34c0e826:::osdmap.19:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7606888) update #-1:ca5b972f:::inc_osdmap.19:0# 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7606888) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:ca5b972f:::inc_osdmap.19:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.613+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92600 allocated 0x[2e000~3000] released 0x[] 2019-12-07T19:33:56.613+0000 7f2b88017700 10 freelist allocate 0x2e000~3000 2019-12-07T19:33:56.613+0000 7f2b88017700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:56.613+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 prepare 2019-12-07T19:33:56.614+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a92600 2019-12-07T19:33:56.614+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 18 ==== pg_create2(e19 {1.cs0=18,2019-12-07T19:33:55.603918+0000}) v2 ==== 207+0+0 (crc 0 0 0) 0x5557e6289200 con 0x5557e6f87000 2019-12-07T19:33:56.614+0000 7f2b967b4700 7 osd.6 18 handle_fast_pg_create pg_create2(e19 {1.cs0=18,2019-12-07T19:33:55.603918+0000}) v2 from mon.0 2019-12-07T19:33:56.614+0000 7f2b967b4700 20 osd.6 18 handle_fast_pg_create 1.cs0 e18@2019-12-07T19:33:55.603918+0000 history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 pi ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.614+0000 7f2b967b4700 15 osd.6 18 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 NullEvt +create_info 2019-12-07T19:33:56.614+0000 7f2b967b4700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.614+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.614+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.614+0000 7f2b744da700 20 osd.6 op_wq _add_slot_waiter 1.cs0 peering, item epoch is 19, will wait on OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.614+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.614+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 aio_wait 2019-12-07T19:33:56.614+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92600 2019-12-07T19:33:56.614+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 io_done 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:56.614+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 7 ==== PGlog(1.3s5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x5557e7a10000 con 0x5557e79cbc00 2019-12-07T19:33:56.614+0000 7f2b957b2700 15 osd.6 18 enqueue_peering_evt 1.3s5 epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info 2019-12-07T19:33:56.614+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.614+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.614+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.614+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 map 19 > 18 2019-12-07T19:33:56.614+0000 7f2b78ce3700 20 osd.6 op_wq _add_slot_waiter 1.3s5 peering, item epoch is 19, will wait on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.614+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3948 < min_flush_size 524288 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606000 had 1 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606780 had 1 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0xf094~2277 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0xf094 0xf094~2277 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545719+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _allocate len 0x130b from 0 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _allocate len 0x130b from 1 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,acd130000 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd0f0000~10000/10000,10000,acd130000 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x20000 to include allocated 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 23 (was clean) 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xf094 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x94 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x30b and padding block with 0xcf5 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x20000 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1130b 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (23) on file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]), flushing log 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 23 len 0x30 crc 0xbed460af) 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x16000~1000 to file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x16000 0x16000~1000 to file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x16000 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x17000 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:56.614+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000016 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 23 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000498119s (0.000070483s flush + 0.000427636s kv commit) 2019-12-07T19:33:56.614+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92600> 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.614+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 kv_submitted 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92600 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7a92600 osr 0x5557e76ce500 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) _discard drop 8 0x1b000~1000 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) prepare_write seq 9 0x1b000~1000 crc 5f344e98 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6 18 _committed_osd_maps 19..19 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) _discard drop 8 0x2000~1000 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6 18 advance to epoch 19 (<= last 19 <= newest_map 19) 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e79ede00) prepare_write seq 9 0x2000~1000 crc 1aeb1035 2019-12-07T19:33:56.614+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.614+0000 7f2b85cfd700 7 osd.6 19 consume_map version 19 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:0.identify_splits_and_merges identify_splits_and_merges slot 1.8s5 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:1.identify_splits_and_merges identify_splits_and_merges slot 1.9s5 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:1.identify_splits_and_merges identify_splits_and_merges slot 1.1s2 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:2.identify_splits_and_merges identify_splits_and_merges slot 1.as1 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:2.identify_splits_and_merges identify_splits_and_merges slot 1.2s3 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs1 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:3.identify_splits_and_merges identify_splits_and_merges slot 1.3s5 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6 19 identify_splits_and_merges 1.cs0 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:5.identify_splits_and_merges identify_splits_and_merges slot 1.5s1 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:6.identify_splits_and_merges identify_splits_and_merges slot 1.es5 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:6.identify_splits_and_merges identify_splits_and_merges slot 1.6s1 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:7.identify_splits_and_merges identify_splits_and_merges slot 1.7s1 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:7.identify_splits_and_merges identify_splits_and_merges slot 1.fs5 has no pg and waiting_for_split 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6 19 prune_pg_created 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6:0.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:0.consume_map consume_map 1.8s5 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:0.consume_map consume_map 1.8s5 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6:1.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.9s5 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.9s5 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.1s2 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.1s2 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6:2.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.as1 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.as1 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.2s3 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.2s3 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6:3.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.bs1 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.bs1 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.3s5 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.3s5 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6:4.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:4.consume_map consume_map 1.cs0 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:4.consume_map consume_map 1.cs0 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:4._wake_pg_slot _wake_pg_slot 1.cs0 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.614+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6:5.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:5.consume_map consume_map 1.5s1 2019-12-07T19:33:56.614+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:5.consume_map consume_map 1.5s1 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6:6.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.es5 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.es5 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.6s1 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.6s1 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b85cfd700 10 osd.6:7.consume_map 19 (was 18) 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.7s1 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.7s1 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.fs5 2019-12-07T19:33:56.614+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.fs5 empty, pruning 2019-12-07T19:33:56.614+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 no pg, should create on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.614+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.614+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.614+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.614+0000 7f2b74cdb700 10 osd.6 19 _make_pg 1.3s5 2019-12-07T19:33:56.614+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.614+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.3(unlocked)] enter Initial 2019-12-07T19:33:56.614+0000 7f2b784e2700 20 osd.6:4.update_pg_epoch 1.cs0 18 -> 19 2019-12-07T19:33:56.614+0000 7f2b784e2700 10 osd.6 pg_epoch: 18 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_advance_map [6,5,4,1,2,3]/[6,5,4,1,2,3] -- 6/6 2019-12-07T19:33:56.614+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Peering advmap 2019-12-07T19:33:56.614+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] adjust_need_up_thru now 18, need_up_thru now false 2019-12-07T19:33:56.614+0000 7f2b74cdb700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Started advmap 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.615+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.cs0_head options 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] hit_set_clear 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] agent_stop 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.3(unlocked)] enter NotTrimming 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.3s5_head fresh osr 0x5557e76cfcc0 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] activate_map 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 5 up [4,1,2,5,7,6] acting [4,1,2,5,7,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.615+0000 7f2b784e2700 7 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: handle ActMap primary 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.615+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] publish_stats_to_osd 0:0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a28310] 2019-12-07T19:33:56.615+0000 7f2b784e2700 5 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:33:56.616283+0000 2 0.000223 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.070790626s + 0s) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Leaving Peering 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.615+0000 7f2b784e2700 5 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering 2019-12-07T19:33:56.616302+0000 0 0.000000 2019-12-07T19:33:56.615+0000 7f2b784e2700 5 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] enter Started/Primary/Active 2019-12-07T19:33:56.615+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] state: In Active, about to call activate 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.3s5_head options 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.616353+0000 0 0.000000 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.070847719s) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(3) 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(3) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(3) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.616398+0000 1 0.000036 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.615+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- PGlog(1.cs3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.070847719s int 16s) e19/18) v6 -- 0x5557e7a11000 con 0x5557e79cac00 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.615+0000 7f2b74cdb700 1 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.616423+0000 0 0.000000 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(3) 1.cs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.2(4) 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.2(4) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7a92300 seq 1 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.3s5_head bits 4 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.2(4) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.3s5_head bits 4 = 0 2019-12-07T19:33:56.615+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- PGlog(1.cs4 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.070847719s int 16s) e19/18) v6 -- 0x5557e7a11800 con 0x5557e7519800 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.3s5_head 0x5557e75b4000) get_onode oid 5#1:c0000000::::head# key 0x858000000000000001c000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.2(4) 1.cs4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.3(5) 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.3s5_head 0x5557e75b4000) r -2 v.len 0 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.3(5) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1068 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.3(5) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._infover' <- _infover 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- PGlog(1.cs5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.070847719s int 16s) e19/18) v6 -- 0x5557e763c000 con 0x5557e79ca800 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.3(5) 1.cs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(2) 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(2) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92300 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92300 onodes 0x5557e795c000 shared_blobs 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e795c108) update 5#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e795c108) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(2) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:c0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.615+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- PGlog(1.cs2 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.070847719s int 16s) e19/18) v6 -- 0x5557e763c800 con 0x5557e79cbc00 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 prepare 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92300 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 io_done 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(2) 1.cs2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(1) 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(1) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 19 handle_pg_create_info new pg pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6:3._attach_pg 1.3s5 0x5557e7954000 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(1) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 19 identify_splits_and_merges 1.3s5 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6:3._prime_splits 2019-12-07T19:33:56.615+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- PGlog(1.cs1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.070847719s int 16s) e19/18) v6 -- 0x5557e763d000 con 0x5557e79ca000 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(1) 1.cs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6:3.update_pg_epoch 1.3s5 18 -> 19 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1130b~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 1(3) 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1130b 0x1130b~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x130b 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x30b 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 2(4) 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x878 and padding block with 0x788 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 3(5) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 4(2) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.3s5_head options 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 5(1) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.615+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 6(0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f2b85cfd700 10 osd.6 19 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.064119817s int 16s) +create_info 2019-12-07T19:33:56.615+0000 7f2b85cfd700 15 osd.6 19 enqueue_peering_evt 1.3s5 epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.615+0000 7f2b85cfd700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) 2019-12-07T19:33:56.615+0000 7f2b85cfd700 15 osd.6 19 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.4(0) 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b85cfd700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) 2019-12-07T19:33:56.615+0000 7f2b85cfd700 7 osd.6 19 activate_map version 19 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 merge_log log((0'0,0'0], crt=0'0) from osd.4(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x11878 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.064119817s int 16s) 2019-12-07T19:33:56.615+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 18446744073.704146016s -> ruub 36.069525417s 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.069525417s 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000125482s (0.000000131s flush + 0.000125351s kv commit) 2019-12-07T19:33:56.615+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.616703+0000 3 0.000056 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.615+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.615+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92300> 2019-12-07T19:33:56.615+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.615+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 kv_submitted 2019-12-07T19:33:56.615+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92300 2019-12-07T19:33:56.615+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92300 finishing 2019-12-07T19:33:56.615+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92300 onodes 0x5557e795c000 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.615+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92300 done 2019-12-07T19:33:56.615+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:56.615+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92300 [] 2019-12-07T19:33:56.615+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.615+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7a92c00 seq 2 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes shared_blobs 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 no pg, should create on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 19 _make_pg 1.bs1 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.b(unlocked)] enter Initial 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.615+0000 7f2b7ace7700 20 osd.6 19 agent_entry empty queue 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.b(unlocked)] enter NotTrimming 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.bs1_head fresh osr 0x5557e79ed2c0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 1 up [4,6,2,0,1,5] acting [4,6,2,0,1,5] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a28310] 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.071434002s + 0s) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.615+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.bs1_head options 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.616959+0000 0 0.000000 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.616984+0000 1 0.000017 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.615+0000 7f2b74cdb700 1 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.616995+0000 0 0.000000 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7a93b00 seq 1 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.bs1_head bits 4 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.bs1_head bits 4 = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.bs1_head 0x5557e75b41e0) get_onode oid 1#1:d0000000::::head# key 0x818000000000000001d000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.bs1_head 0x5557e75b41e0) r -2 v.len 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1069 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._infover' <- _infover 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93b00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93b00 onodes 0x5557e795c280 shared_blobs 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e795c388) update 1#1:d0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e795c388) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:d0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 prepare 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93b00 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 io_done 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 19 handle_pg_create_info new pg pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6:3._attach_pg 1.bs1 0x5557e7957000 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 19 identify_splits_and_merges 1.bs1 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6:3._prime_splits 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6:3.update_pg_epoch 1.bs1 18 -> 19 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 18 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.bs1_head options 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.4(0) 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 merge_log log((0'0,0'0], crt=0'0) from osd.4(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.063561597s int 16s) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 18446744073.704146016s -> ruub 36.068967197s 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.068967197s 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.617140+0000 3 0.000023 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.615+0000 7f2b74cdb700 5 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7630000 seq 2 2019-12-07T19:33:56.615+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:56.615+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630000 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630000 onodes shared_blobs 2019-12-07T19:33:56.615+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.616+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 prepare 2019-12-07T19:33:56.616+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630000 2019-12-07T19:33:56.616+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 io_done 2019-12-07T19:33:56.616+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.616+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) pg 0x5557e7954000 2019-12-07T19:33:56.616+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 5 ==== PGlog(1.5s1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.087862669s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x5557e7a10800 con 0x5557e7519800 2019-12-07T19:33:56.616+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.616+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b95fb3700 15 osd.6 19 enqueue_peering_evt 1.5s1 epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.087862669s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.087862669s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.616+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.087862669s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.087862669s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.615+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] release_backoffs [1:30000000::::0,1:40000000::::head) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 no pg, should create on OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.087862669s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 19 _make_pg 1.5s1 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 6 ==== PGlog(1.fs5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x5557e763d000 con 0x5557e79ca800 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 18 pg[1.5(unlocked)] enter Initial 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.fs5 epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.616+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 18 pg[1.5(unlocked)] enter NotTrimming 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 no pg, should create on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.5s1_head fresh osr 0x5557e75c8280 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.616+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 18:1 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 1 up [2,6,3,4,1,5] acting [2,6,3,4,1,5] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: Activate Finished 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.616+0000 7f2b784e2700 5 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] enter Started/Primary/Active/Activating 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a283f0] 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 19 _make_pg 1.fs5 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.071917913s + 0s) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 18 pg[1.f(unlocked)] enter Initial 2019-12-07T19:33:56.616+0000 7f2b77ce1700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 NullEvt +create_info 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7630f00 seq 3 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.616+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.616+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.5s1_head options 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630f00 cost 5345 (1 ios * 4000 + 1345 bytes) 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630f00 onodes shared_blobs 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.617466+0000 0 0.000000 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 prepare 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630f00 2019-12-07T19:33:56.616+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 io_done 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 18 pg[1.f(unlocked)] enter NotTrimming 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.fs5_head fresh osr 0x5557e75c83c0 2019-12-07T19:33:56.616+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 5 up [3,0,5,4,1,6] acting [3,0,5,4,1,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.616+0000 7f2b744da700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.616+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.617504+0000 1 0.000029 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.616+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a29110] 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.072023331s + 0s) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 1 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.617526+0000 0 0.000000 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.616+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7631800 seq 1 2019-12-07T19:33:56.616+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.5s1_head bits 4 2019-12-07T19:33:56.616+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.fs5_head options 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.5s1_head bits 4 = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.5s1_head 0x5557e6281c20) get_onode oid 1#1:a0000000::::head# key 0x818000000000000001a000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.617561+0000 0 0.000000 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.5s1_head 0x5557e6281c20) r -2 v.len 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.616+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1070 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.616+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._infover' <- _infover 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.616+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.617603+0000 1 0.000033 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631800 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631800 onodes 0x5557e795c780 shared_blobs 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557e795c888) update 1#1:a0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557e795c888) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:a0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.616+0000 7f2b72cd7700 1 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 prepare 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631800 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 io_done 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.617624+0000 0 0.000000 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 19 handle_pg_create_info new pg pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6:5._attach_pg 1.5s1 0x5557e764c000 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6:5._wake_pg_slot _wake_pg_slot 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 19 identify_splits_and_merges 1.5s1 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6:5._prime_splits 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7664000 seq 1 2019-12-07T19:33:56.616+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.fs5_head bits 4 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.fs5_head bits 4 = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6:5.update_pg_epoch 1.5s1 18 -> 19 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.fs5_head 0x5557e75e32c0) get_onode oid 5#1:f0000000::::head# key 0x858000000000000001f000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 18 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [2,6,3,4,1,5]/[2,6,3,4,1,5] -- 2/2 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.fs5_head 0x5557e75e32c0) r -2 v.len 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.616+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.5s1_head options 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1071 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._infover' <- _infover 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.616+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664000 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664000 onodes 0x5557e795ca00 shared_blobs 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e795cb08) update 5#1:f0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e795cb08) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.087862669s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:f0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.2(0) 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 merge_log log((0'0,0'0], crt=0'0) from osd.2(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664000 prepare 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664000 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664000 io_done 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.087862669s int 16s) 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.017127956s -> ruub 36.070734713s 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 19 handle_pg_create_info new pg pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6:7._attach_pg 1.fs5 0x5557e764f000 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.070734713s 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 19 identify_splits_and_merges 1.fs5 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6:7._prime_splits 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.617766+0000 3 0.000046 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f2b77ce1700 5 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6:7.update_pg_epoch 1.fs5 18 -> 19 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 2 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 18 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3570 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4758 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:56.616+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.fs5_head options 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6124 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 7490 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7631b00 seq 2 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 7513 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 7513 < min_flush_size 524288 2019-12-07T19:33:56.616+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x11878~1d59 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.616+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x11878 0x11878~1d59 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x1878 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x878 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.616+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x5d1 and padding block with 0xa2f 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631b00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631b00 onodes shared_blobs 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.3(0) 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 merge_log log((0'0,0'0], crt=0'0) from osd.3(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 prepare 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631b00 2019-12-07T19:33:56.616+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 io_done 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074133487s int 16s) 2019-12-07T19:33:56.616+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002940140s -> ruub 36.071193347s 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.071193347s 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.617873+0000 3 0.000052 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.616+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f2b72cd7700 5 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7664300 seq 2 2019-12-07T19:33:56.616+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664300 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664300 onodes shared_blobs 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 prepare 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664300 2019-12-07T19:33:56.616+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 io_done 2019-12-07T19:33:56.616+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 9 ==== PGlog(1.9s5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x5557e763c800 con 0x5557e79cac00 2019-12-07T19:33:56.616+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.9s5 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 10 ==== PGlog(1.es5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x5557e763c000 con 0x5557e79cac00 2019-12-07T19:33:56.616+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.es5 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 no pg, should create on OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 19 _make_pg 1.es5 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 18 pg[1.e(unlocked)] enter Initial 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 18 pg[1.e(unlocked)] enter NotTrimming 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.es5_head fresh osr 0x5557e75c9a40 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 5 up [1,3,4,5,0,6] acting [1,3,4,5,0,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a91ea0] 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.072898839s + 0s) 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.617+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.es5_head options 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.618437+0000 0 0.000000 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.618480+0000 1 0.000033 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.617+0000 7f2b734d8700 1 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.618500+0000 0 0.000000 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e7664900 seq 1 2019-12-07T19:33:56.617+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.es5_head bits 4 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.es5_head bits 4 = 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.es5_head 0x5557e75e34a0) get_onode oid 5#1:70000000::::head# key 0x8580000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.es5_head 0x5557e75e34a0) r -2 v.len 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1072 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._infover' <- _infover 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664900 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664900 onodes 0x5557e795cc80 shared_blobs 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e795cd88) update 5#1:70000000::::head# 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e795cd88) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:70000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 prepare 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664900 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 io_done 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 19 handle_pg_create_info new pg pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6:6._attach_pg 1.es5 0x5557e766a000 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6:6._wake_pg_slot _wake_pg_slot 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 19 identify_splits_and_merges 1.es5 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6:6._prime_splits 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6:6.update_pg_epoch 1.es5 18 -> 19 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 18 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.617+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.es5_head options 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074554249s int 16s) 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002396109s -> ruub 36.072158140s 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.072158140s 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.618720+0000 3 0.000042 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.617+0000 7f2b734d8700 5 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e7664c00 seq 2 2019-12-07T19:33:56.617+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664c00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664c00 onodes shared_blobs 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664c00 prepare 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664c00 2019-12-07T19:33:56.617+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664c00 io_done 2019-12-07T19:33:56.617+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 11 ==== PGlog(1.as1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x5557e7a10000 con 0x5557e79cac00 2019-12-07T19:33:56.617+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.as1 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info 2019-12-07T19:33:56.617+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 no pg, should create on OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 19 _make_pg 1.as1 2019-12-07T19:33:56.617+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.a(unlocked)] enter Initial 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.a(unlocked)] enter NotTrimming 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.as1_head fresh osr 0x5557e75c9e00 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 1 up [1,6,2,0,5,4] acting [1,6,2,0,5,4] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a91ea0] 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.073546874s + 0s) 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.617+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.617+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.as1_head options 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.617+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.619079+0000 0 0.000000 2019-12-07T19:33:56.617+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.617+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.619117+0000 1 0.000029 2019-12-07T19:33:56.617+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.617+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.617+0000 7f2b754dc700 1 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.617+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.619138+0000 0 0.000000 2019-12-07T19:33:56.617+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e7665200 seq 1 2019-12-07T19:33:56.617+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.as1_head bits 4 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.as1_head bits 4 = 0 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.as1_head 0x5557e75e2d20) get_onode oid 1#1:50000000::::head# key 0x8180000000000000015000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.as1_head 0x5557e75e2d20) r -2 v.len 0 2019-12-07T19:33:56.617+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:56.617+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1073 2019-12-07T19:33:56.617+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._infover' <- _infover 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665200 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665200 onodes 0x5557e795cf00 shared_blobs 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e795d008) update 1#1:50000000::::head# 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e795d008) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:50000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 prepare 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665200 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 io_done 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 19 handle_pg_create_info new pg pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6:2._attach_pg 1.as1 0x5557e766d000 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6:2._wake_pg_slot _wake_pg_slot 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 19 identify_splits_and_merges 1.as1 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6:2._prime_splits 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6:2.update_pg_epoch 1.as1 18 -> 19 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 no pg, should create on OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.618+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.as1_head options 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 19 _make_pg 1.9s5 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.9(unlocked)] enter Initial 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.075332677s int 16s) 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002396109s -> ruub 36.072936568s 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.072936568s 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7f2b754dc700 5 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.619363+0000 3 0.000042 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7f2b754dc700 5 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.618+0000 7f2b754dc700 5 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.9(unlocked)] enter NotTrimming 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.9s5_head fresh osr 0x5557e7678280 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 5 up [1,4,5,0,7,6] acting [1,4,5,0,7,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a91ea0] 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.073923087s + 0s) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e7665500 seq 2 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.618+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665500 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665500 onodes shared_blobs 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 prepare 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665500 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 io_done 2019-12-07T19:33:56.618+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.9s5_head options 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.619463+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.619500+0000 1 0.000028 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.618+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 3 ==== PGlog(1.2s3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x5557e70a7000 con 0x5557e79cb800 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.618+0000 7f2b75cdd700 1 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.618+0000 7f2b967b4700 15 osd.6 19 enqueue_peering_evt 1.2s3 epoch_sent: 19 epoch_requested: 18 MLogRec from 0(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) +create_info 2019-12-07T19:33:56.618+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 0(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.619520+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7665b00 seq 1 2019-12-07T19:33:56.618+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.9s5_head bits 4 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.9s5_head bits 4 = 0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.9s5_head 0x5557e75e2b40) get_onode oid 5#1:90000000::::head# key 0x8580000000000000019000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.9s5_head 0x5557e75e2b40) r -2 v.len 0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1074 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._infover' <- _infover 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665b00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665b00 onodes 0x5557e795d180 shared_blobs 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e795d288) update 5#1:90000000::::head# 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e795d288) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:90000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665b00 prepare 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665b00 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665b00 io_done 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 19 handle_pg_create_info new pg pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6:1._attach_pg 1.9s5 0x5557e767c000 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6:1._wake_pg_slot _wake_pg_slot 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 19 identify_splits_and_merges 1.9s5 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6:1._prime_splits 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6:1.update_pg_epoch 1.9s5 18 -> 19 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.618+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.9s5_head options 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074537935s int 16s) 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002396109s -> ruub 36.072141826s 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.072141826s 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.619743+0000 3 0.000042 2019-12-07T19:33:56.618+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.618+0000 7f2b75cdd700 5 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7692000 seq 2 2019-12-07T19:33:56.618+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692000 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692000 onodes shared_blobs 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 prepare 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692000 2019-12-07T19:33:56.618+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 io_done 2019-12-07T19:33:56.618+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x135d1 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 6 cleaned 0 in 0.002103691s (0.000000101s flush + 0.002103590s kv commit) 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 8 submitting 8 deferred done 0 stable 0 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2204 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3570 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 2 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4672 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6038 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 2 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 7140 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 1 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8506 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 2 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9608 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9631 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9631 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x135d1~259f to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x135d1 0x135d1~259f to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x35d1 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x5d1 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92c00,0x5557e7a93b00,0x5557e7630000,0x5557e7630f00,0x5557e7631800,0x5557e7664000> 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xb70 and padding block with 0x490 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93b00 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 finishing 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93b00 onodes 0x5557e795c280 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93b00 done 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630000 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93b00 [] 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630000 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 finishing 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630000 onodes 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630000 done 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630000 [] 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630f00 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 finishing 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630f00 onodes 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630f00 done 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630f00 [] 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631800 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 finishing 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631800 onodes 0x5557e795c780 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631800 done 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631b00 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631800 [] 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664000 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664000 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664000 finishing 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664000 onodes 0x5557e795ca00 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664000 done 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664300 kv_submitted 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664000 [] 2019-12-07T19:33:56.618+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.618+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 0(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 0(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.618+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 no pg, should create on OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 0(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 19 _make_pg 1.2s3 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x15b70 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.618+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.618+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.2(unlocked)] enter Initial 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 8 cleaned 0 in 0.000221024s (0.000000140s flush + 0.000220884s kv commit) 2019-12-07T19:33:56.618+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.618+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.618+0000 7f2b78ce3700 15 osd.6 19 enqueue_peering_evt 1.3s5 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7f2b78ce3700 15 osd.6 19 enqueue_peering_evt 1.bs1 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e7954000 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.2(unlocked)] enter NotTrimming 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.2s3_head fresh osr 0x5557e7678000 2019-12-07T19:33:56.618+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 3 up [0,4,1,6,5,3] acting [0,4,1,6,5,3] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.618+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.618+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.618+0000 7f2b78ce3700 10 osd.6 19 send_incremental_map 18 -> 19 to 0x5557e79cbc00 v2:172.21.15.109:6810/36938 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a28150] 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.074671456s + 0s) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.618+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~0 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~285 size 0x285 (645) 2019-12-07T19:33:56.618+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:56.618+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e74d98f0 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x5557e7a28a10 sbid 0x0)) need 0x0~285 cache has 0x[0~285] 2019-12-07T19:33:56.618+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.618+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 = 645 2019-12-07T19:33:56.618+0000 7f2b78ce3700 10 osd.6 19 add_map_inc_bl 19 645 bytes 2019-12-07T19:33:56.619+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.2s3_head options 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.619+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(19..19 src has 1..19) v4 -- 0x5557e7adeb40 con 0x5557e79cbc00 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.620208+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7f2b78ce3700 10 osd.6 19 maybe_share_map con 0x5557e79cbc00 v2:172.21.15.109:6810/36938 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.619+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.3s0 1.3s0 1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e70b6300 con 0x5557e79cbc00 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.619+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.619+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e7957000 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.620245+0000 1 0.000029 2019-12-07T19:33:56.619+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.619+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e70b6a00 con 0x5557e79cbc00 2019-12-07T19:33:56.619+0000 7f2b754dc700 1 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.619+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.620266+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7631b00,0x5557e7664300,0x5557e7664900,0x5557e7664c00,0x5557e7665200,0x5557e7665500,0x5557e7665b00,0x5557e7692000> 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631b00 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 finishing 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631b00 onodes 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631b00 done 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631b00 [] 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664300 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 finishing 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e7a92c00 seq 1 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664300 onodes 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664300 done 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:56.619+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.2s3_head bits 4 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664300 [] 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664900 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.2s3_head bits 4 = 0 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 finishing 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664900 onodes 0x5557e795cc80 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664900 done 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664c00 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664900 [] 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664c00 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664c00 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664c00 finishing 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.2s3_head 0x5557e75b4d20) get_onode oid 3#1:40000000::::head# key 0x8380000000000000014000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664c00 onodes 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664c00 done 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664c00 [] 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665200 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 finishing 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665200 onodes 0x5557e795cf00 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665200 done 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665500 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665200 [] 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.2s3_head 0x5557e75b4d20) r -2 v.len 0 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665500 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 finishing 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665500 onodes 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665500 done 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9e00 q now empty 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665500 [] 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.619+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1075 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665b00 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665b00 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665b00 finishing 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665b00 onodes 0x5557e795d180 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665b00 done 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692000 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665b00 [] 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._infover' <- _infover 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 kv_submitted 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692000 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 finishing 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692000 onodes 2019-12-07T19:33:56.619+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692000 done 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692000 [] 2019-12-07T19:33:56.619+0000 7f2b77ce1700 15 osd.6 19 enqueue_peering_evt 1.5s1 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.619+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:56.619+0000 7f2b77ce1700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:56.619+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.619+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.619+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.619+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e764c000 2019-12-07T19:33:56.619+0000 7f2b774e0700 15 osd.6 19 enqueue_peering_evt 1.es5 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes 0x5557e795d400 shared_blobs 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e795d508) update 3#1:40000000::::head# 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e795d508) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.619+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:40000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e766a000 2019-12-07T19:33:56.619+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:56.619+0000 7f2b77ce1700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b77ce1700 10 osd.6 19 send_incremental_map 18 -> 19 to 0x5557e7519800 v2:172.21.15.131:6804/37077 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 19 handle_pg_create_info new pg pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6:2._attach_pg 1.2s3 0x5557e767f000 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6:2._wake_pg_slot _wake_pg_slot 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 19 identify_splits_and_merges 1.2s3 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6:2._prime_splits 2019-12-07T19:33:56.619+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(19..19 src has 1..19) v4 -- 0x5557e7666000 con 0x5557e7519800 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 19 send_incremental_map 18 -> 19 to 0x5557e79cac00 v2:172.21.15.131:6823/37078 2019-12-07T19:33:56.619+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6:2.update_pg_epoch 1.2s3 18 -> 19 2019-12-07T19:33:56.619+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(19..19 src has 1..19) v4 -- 0x5557e7adf440 con 0x5557e79cac00 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 18 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 19 maybe_share_map con 0x5557e79cac00 v2:172.21.15.131:6823/37078 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.619+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.es0 1.es0 1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7698000 con 0x5557e79cac00 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.619+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.2s3_head options 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.619+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 0(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) +create_info 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.0(0) 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 merge_log log((0'0,0'0], crt=0'0) from osd.0(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.073596236s int 16s) 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 18446744073.709452256s -> ruub 36.073695596s 2019-12-07T19:33:56.619+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.073695596s 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.620482+0000 3 0.000044 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.619+0000 7f2b754dc700 5 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e7046f00 seq 2 2019-12-07T19:33:56.619+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes shared_blobs 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:56.619+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:56.619+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b76cdf700 15 osd.6 19 enqueue_peering_evt 1.fs5 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b76cdf700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f2b77ce1700 10 osd.6 19 maybe_share_map con 0x5557e7519800 v2:172.21.15.131:6804/37077 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.619+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e764f000 2019-12-07T19:33:56.619+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.5s0 1.5s0 1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e70b7800 con 0x5557e7519800 2019-12-07T19:33:56.619+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.619+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b76cdf700 10 osd.6 19 send_incremental_map 18 -> 19 to 0x5557e79ca800 v2:172.21.15.131:6812/37076 2019-12-07T19:33:56.619+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(19..19 src has 1..19) v4 -- 0x5557e74cb8c0 con 0x5557e79ca800 2019-12-07T19:33:56.619+0000 7f2b76cdf700 10 osd.6 19 maybe_share_map con 0x5557e79ca800 v2:172.21.15.131:6812/37076 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.619+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.fs0 1.fs0 1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7698700 con 0x5557e79ca800 2019-12-07T19:33:56.619+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 7 ==== PGlog(1.1s2 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x5557e7a0e000 con 0x5557e79ca800 2019-12-07T19:33:56.619+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.1s2 epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info 2019-12-07T19:33:56.619+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 8 ==== PGlog(1.6s1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) e19/18) v6 ==== 1123+0+0 (crc 0 0 0) 0x5557e763d800 con 0x5557e79ca800 2019-12-07T19:33:56.619+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.6s1 epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info 2019-12-07T19:33:56.619+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 no pg, should create on OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 19 _make_pg 1.6s1 2019-12-07T19:33:56.619+0000 7f2b774e0700 5 osd.6 pg_epoch: 18 pg[1.6(unlocked)] enter Initial 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 pg_epoch: 18 pg[1.6(unlocked)] enter NotTrimming 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.6s1_head fresh osr 0x5557e7678640 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 1 up [3,6,7,1,2,5] acting [3,6,7,1,2,5] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a29110] 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.075469716s + 0s) 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.619+0000 7f2b774e0700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.6s1_head options 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.619+0000 7f2b774e0700 5 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.621001+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7f2b774e0700 5 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7f2b774e0700 5 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.621039+0000 1 0.000029 2019-12-07T19:33:56.619+0000 7f2b784e2700 15 osd.6 19 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b784e2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7f2b774e0700 5 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.619+0000 7f2b774e0700 5 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.619+0000 7f2b774e0700 1 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.619+0000 7f2b774e0700 5 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.621059+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b774e0700 5 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.619+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.619+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7664300 seq 1 2019-12-07T19:33:56.619+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.6s1_head bits 4 2019-12-07T19:33:56.619+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.619+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: _activate_committed 19 peer_activated now 6(0) last_interval_started 0 last_epoch_started 0 same_interval_since 18 2019-12-07T19:33:56.619+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.6s1_head bits 4 = 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) get_onode oid 1#1:60000000::::head# key 0x8180000000000000016000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) r -2 v.len 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1076 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._infover' <- _infover 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664300 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664300 onodes 0x5557e795d900 shared_blobs 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e795da08) update 1#1:60000000::::head# 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e795da08) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:60000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 prepare 2019-12-07T19:33:56.619+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664300 2019-12-07T19:33:56.619+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 io_done 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 19 handle_pg_create_info new pg pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6:6._attach_pg 1.6s1 0x5557e769c000 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6:6._wake_pg_slot _wake_pg_slot 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6 19 identify_splits_and_merges 1.6s1 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6:6._prime_splits 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6:6.update_pg_epoch 1.6s1 18 -> 19 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 18 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.620+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.6s1_head options 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075844307s int 16s) +create_info 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.3(0) 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 merge_log log((0'0,0'0], crt=0'0) from osd.3(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.075844307s int 16s) 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002940140s -> ruub 36.072904167s 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.072904167s 2019-12-07T19:33:56.620+0000 7f2b774e0700 5 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.621331+0000 3 0.000044 2019-12-07T19:33:56.620+0000 7f2b774e0700 5 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.620+0000 7f2b774e0700 5 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7664900 seq 2 2019-12-07T19:33:56.620+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664900 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664900 onodes shared_blobs 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 prepare 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664900 2019-12-07T19:33:56.620+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 io_done 2019-12-07T19:33:56.620+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 9 ==== pg_info2(1.cs0 1.cs0 1.cs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (crc 0 0 0) 0x5557e70b6300 con 0x5557e79ca800 2019-12-07T19:33:56.620+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.cs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.620+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.cs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.cs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.cs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.620+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 10 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (crc 0 0 0) 0x5557e74cb8c0 con 0x5557e79ca800 2019-12-07T19:33:56.620+0000 7f2b834f8700 20 osd.6 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.620+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.cs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.620+0000 7f2b834f8700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:56.620+0000 7f2b834f8700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:56.620+0000 7f2b834f8700 20 osd.6 19 _dispatch 0x5557e74cb8c0 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.620+0000 7f2b834f8700 3 osd.6 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.620+0000 7f2b834f8700 10 osd.6 19 no new maps here, dropping 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.070847719s 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.070847719s) 2019-12-07T19:33:56.620+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.3(5) activated and committed 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 no pg, should create on OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.620+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:2 2019-12-07T19:33:56.620+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 19 _make_pg 1.1s2 2019-12-07T19:33:56.620+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.1(unlocked)] enter Initial 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.1(unlocked)] enter NotTrimming 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.1s2_head fresh osr 0x5557e76783c0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 2 up [3,0,6,4,5,1] acting [3,0,6,4,5,1] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a29110] 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.076371772s + 0s) 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.620+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.1s2_head options 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.620+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.621908+0000 0 0.000000 2019-12-07T19:33:56.620+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.620+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.621952+0000 1 0.000035 2019-12-07T19:33:56.620+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.620+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.620+0000 7f2b75cdd700 1 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.620+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.621974+0000 0 0.000000 2019-12-07T19:33:56.620+0000 7f2b75cdd700 5 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e7692600 seq 1 2019-12-07T19:33:56.620+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.1s2_head bits 4 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.1s2_head bits 4 = 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.1s2_head 0x5557e76a8000) get_onode oid 2#1:80000000::::head# key 0x8280000000000000018000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.1s2_head 0x5557e76a8000) r -2 v.len 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1077 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._infover' <- _infover 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._biginfo' <- _biginfo 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._epoch' <- _epoch 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692600 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692600 onodes 0x5557e795db80 shared_blobs 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e795dc88) update 2#1:80000000::::head# 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e795dc88) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 2#1:80000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692600 allocated 0x[] released 0x[] 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 prepare 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692600 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 io_done 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 19 handle_pg_create_info new pg pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6:1._attach_pg 1.1s2 0x5557e769f000 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6:1._wake_pg_slot _wake_pg_slot 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 19 identify_splits_and_merges 1.1s2 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6:1._prime_splits 2019-12-07T19:33:56.620+0000 7f2b79ce5700 15 osd.6 19 enqueue_peering_evt 1.9s5 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.620+0000 7f2b79ce5700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.620+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.620+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.620+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:33:56.620+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e767c000 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6:1.update_pg_epoch 1.1s2 18 -> 19 2019-12-07T19:33:56.620+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.620+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 18 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:33:56.620+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.620+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.620+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 2 2019-12-07T19:33:56.620+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7698e00 con 0x5557e79cac00 2019-12-07T19:33:56.620+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.1s2_head options 2019-12-07T19:33:56.620+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.620+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.620+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.620+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3834 < min_flush_size 524288 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.620+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 2 2019-12-07T19:33:56.620+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4936 < min_flush_size 524288 2019-12-07T19:33:56.620+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.3(0) 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 merge_log log((0'0,0'0], crt=0'0) from osd.3(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6302 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.621+0000 7f2b794e4700 15 osd.6 19 enqueue_peering_evt 1.as1 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074132637s int 16s) 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002940140s -> ruub 36.071192497s 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.071192497s 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6325 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6325 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.621+0000 7f2b75cdd700 5 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.622221+0000 3 0.000041 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7f2b75cdd700 5 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e766d000 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x15b70~18b5 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7f2b75cdd700 5 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x15b70 0x15b70~18b5 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.621+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x5b70 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xb70 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x425 and padding block with 0xbdb 2019-12-07T19:33:56.621+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.621+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e7630f00 seq 2 2019-12-07T19:33:56.621+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._biginfo' <- _biginfo 2019-12-07T19:33:56.621+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.as0 1.as0 1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7699500 con 0x5557e79cac00 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._epoch' <- _epoch 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630f00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630f00 onodes shared_blobs 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 prepare 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630f00 2019-12-07T19:33:56.621+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 io_done 2019-12-07T19:33:56.621+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x17425 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 5 cleaned 0 in 0.000152705s (0.000000086s flush + 0.000152619s kv commit) 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92c00,0x5557e7046f00,0x5557e7664300,0x5557e7664900,0x5557e7692600> 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 0x5557e795d400 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:56.621+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:56.621+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x17425~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x17425 0x17425~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664300 2019-12-07T19:33:56.621+0000 7f2b794e4700 15 osd.6 19 enqueue_peering_evt 1.2s3 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 finishing 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664300 onodes 0x5557e795d900 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664300 done 2019-12-07T19:33:56.621+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664900 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x7425 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664300 [] 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x425 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664900 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x88a and padding block with 0x776 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 finishing 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664900 onodes 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664900 done 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664900 [] 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e767f000 2019-12-07T19:33:56.621+0000 7f2b774e0700 15 osd.6 19 enqueue_peering_evt 1.6s1 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692600 2019-12-07T19:33:56.621+0000 7f2b774e0700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 finishing 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692600 onodes 0x5557e795db80 2019-12-07T19:33:56.621+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692600 done 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630f00 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692600 [] 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.621+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.621+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.621+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e769c000 2019-12-07T19:33:56.621+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.621+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.621+0000 7f2b794e4700 10 osd.6 19 send_incremental_map 18 -> 19 to 0x5557e79cb800 v2:172.21.15.131:6822/37079 2019-12-07T19:33:56.621+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.621+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(19..19 src has 1..19) v4 -- 0x5557e7adf8c0 con 0x5557e79cb800 2019-12-07T19:33:56.621+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.6s0 1.6s0 1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e769aa00 con 0x5557e79ca800 2019-12-07T19:33:56.621+0000 7f2b794e4700 10 osd.6 19 maybe_share_map con 0x5557e79cb800 v2:172.21.15.131:6822/37079 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.621+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- pg_info2(1.2s0 1.2s0 1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7699c00 con 0x5557e79cb800 2019-12-07T19:33:56.621+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1788a 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.621+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000127574s (0.000000065s flush + 0.000127509s kv commit) 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.621+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630f00> 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 kv_submitted 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630f00 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 finishing 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630f00 onodes 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630f00 done 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630f00 [] 2019-12-07T19:33:56.621+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b79ce5700 15 osd.6 19 enqueue_peering_evt 1.1s2 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.621+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.621+0000 7f2b79ce5700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.621+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.621+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e769f000 2019-12-07T19:33:56.621+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.621+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.621+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.1s0 1.1s0 1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e769b800 con 0x5557e79ca800 2019-12-07T19:33:56.621+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 12 ==== PGlog(1.7s1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x5557e763e000 con 0x5557e79cac00 2019-12-07T19:33:56.621+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.7s1 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info 2019-12-07T19:33:56.621+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 13 ==== PGlog(1.8s5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.078095466s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x5557e763e800 con 0x5557e79cac00 2019-12-07T19:33:56.622+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.8s5 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.078095466s int 16s) +create_info 2019-12-07T19:33:56.622+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.078095466s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.078095466s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.078095466s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 no pg, should create on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.078095466s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 19 _make_pg 1.8s5 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 18 pg[1.8(unlocked)] enter Initial 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 18 pg[1.8(unlocked)] enter NotTrimming 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.8s5_head fresh osr 0x5557e7678a00 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 5 up [1,4,5,2,3,6] acting [1,4,5,2,3,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a91ea0] 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.077901440s + 0s) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.8s5_head options 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.623451+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.623493+0000 1 0.000033 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 1 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.623516+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7665200 seq 1 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.8s5_head bits 4 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.8s5_head bits 4 = 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:10000000::::head# key 0x8580000000000000011000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) r -2 v.len 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1078 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._infover' <- _infover 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._biginfo' <- _biginfo 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._epoch' <- _epoch 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665200 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665200 onodes 0x5557e76ca000 shared_blobs 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e76ca108) update 5#1:10000000::::head# 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e76ca108) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:10000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 prepare 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665200 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 io_done 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 19 handle_pg_create_info new pg pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6:0._attach_pg 1.8s5 0x5557e76ba000 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 19 identify_splits_and_merges 1.8s5 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6:0._prime_splits 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6:0.update_pg_epoch 1.8s5 18 -> 19 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 18 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,4,5,2,3,6]/[1,4,5,2,3,6] -- 1/1 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1788a~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1788a 0x1788a~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x788a 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.8s5_head options 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x88a 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xdf7 and padding block with 0x209 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.078095466s int 16s) +create_info 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.078095466s int 16s) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002396109s -> ruub 36.075699357s 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.075699357s 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.623745+0000 3 0.000047 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 5 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 no pg, should create on OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 19 _make_pg 1.7s1 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7665500 seq 2 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._biginfo' <- _biginfo 2019-12-07T19:33:56.622+0000 7f2b76cdf700 5 osd.6 pg_epoch: 18 pg[1.7(unlocked)] enter Initial 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._epoch' <- _epoch 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665500 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665500 onodes shared_blobs 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 prepare 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665500 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 io_done 2019-12-07T19:33:56.622+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 pg_epoch: 18 pg[1.7(unlocked)] enter NotTrimming 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _osr_attach 1.7s1_head fresh osr 0x5557e7678dc0 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x17df7 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 1 up [1,6,4,0,7,2] acting [1,6,4,0,7,2] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000239747s (0.000000096s flush + 0.000239651s kv commit) 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x5557e7a91ea0] 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 521 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.078390036s + 0s) 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1113 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1136 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1136 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x17df7~470 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x17df7 0x17df7~470 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x7df7 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xdf7 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x267 and padding block with 0xd99 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x18267 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.622+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000085569s (0.000000097s flush + 0.000085472s kv commit) 2019-12-07T19:33:56.622+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.622+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.7s1_head options 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.622+0000 7f2b76cdf700 5 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.623986+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7f2b76cdf700 5 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.622+0000 7f2b76cdf700 5 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.624023+0000 1 0.000028 2019-12-07T19:33:56.622+0000 7f2b76cdf700 5 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.622+0000 7f2b76cdf700 5 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.622+0000 7f2b76cdf700 1 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.622+0000 7f2b76cdf700 5 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.624043+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7f2b76cdf700 5 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7046f00 seq 1 2019-12-07T19:33:56.622+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.7s1_head bits 4 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _create_collection 1.7s1_head bits 4 = 0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.7s1_head 0x5557e76a92c0) get_onode oid 1#1:e0000000::::head# key 0x818000000000000001e000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.7s1_head 0x5557e76a92c0) r -2 v.len 0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1079 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._infover' <- _infover 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._biginfo' <- _biginfo 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._epoch' <- _epoch 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes 0x5557e76ca280 shared_blobs 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore.extentmap(0x5557e76ca388) update 1#1:e0000000::::head# 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore.extentmap(0x5557e76ca388) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:e0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 19 handle_pg_create_info new pg pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6:7._attach_pg 1.7s1 0x5557e76bd000 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6:7._wake_pg_slot _wake_pg_slot 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7f2b76cdf700 20 osd.6 19 identify_splits_and_merges 1.7s1 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6:7._prime_splits 2019-12-07T19:33:56.622+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.622+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.622+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7665200,0x5557e7665500> 2019-12-07T19:33:56.622+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 kv_submitted 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665200 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 finishing 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665200 onodes 0x5557e76ca000 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665200 done 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665500 kv_submitted 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665200 [] 2019-12-07T19:33:56.623+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 kv_submitted 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665500 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 finishing 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665500 onodes 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665500 done 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665500 [] 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6:7.update_pg_epoch 1.7s1 18 -> 19 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 18 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.623+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.7s1_head options 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.075985074s int 16s) 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002396109s -> ruub 36.073588965s 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.073588965s 2019-12-07T19:33:56.623+0000 7f2b76cdf700 5 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.624266+0000 3 0.000040 2019-12-07T19:33:56.623+0000 7f2b76cdf700 5 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.623+0000 7f2b76cdf700 5 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7a92c00 seq 2 2019-12-07T19:33:56.623+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._biginfo' <- _biginfo 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._epoch' <- _epoch 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes shared_blobs 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.623+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 8 ==== pg_info2(1.cs0 1.cs0 1.cs2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (crc 0 0 0) 0x5557e7699500 con 0x5557e79cbc00 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(2) info: 1.cs2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.623+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(2) info: 1.cs2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 2 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x18267~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x18267 0x18267~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x8267 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x267 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xc22 and padding block with 0x3de 2019-12-07T19:33:56.623+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 9 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (crc 0 0 0) 0x5557e7adf440 con 0x5557e79cbc00 2019-12-07T19:33:56.623+0000 7f2b834f8700 20 osd.6 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.623+0000 7f2b834f8700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:56.623+0000 7f2b834f8700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:56.623+0000 7f2b834f8700 20 osd.6 19 _dispatch 0x5557e7adf440 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.623+0000 7f2b834f8700 3 osd.6 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.623+0000 7f2b834f8700 10 osd.6 19 no new maps here, dropping 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(2) info: 1.cs2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(2) info: 1.cs2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.623+0000 7f2b744da700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(2) info: 1.cs2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.070847719s 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 0s 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.070847719s 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.070847719s) 2019-12-07T19:33:56.623+0000 7f2b744da700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.4(2) activated and committed 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x18c22 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.623+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000171040s (0.000000082s flush + 0.000170958s kv commit) 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00,0x5557e7a92c00> 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 0x5557e76ca280 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 15 osd.6 19 enqueue_peering_evt 1.8s5 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:56.623+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.623+0000 7f2b76cdf700 15 osd.6 19 enqueue_peering_evt 1.7s1 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.623+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e76ba000 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x5557e76bd000 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.623+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.623+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.623+0000 7f2b744da700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:3 2019-12-07T19:33:56.623+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7698700 con 0x5557e79cac00 2019-12-07T19:33:56.623+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.8s0 1.8s0 1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e70b6a00 con 0x5557e79cac00 2019-12-07T19:33:56.623+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 6 ==== pg_info2(1.cs0 1.cs0 1.cs4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x5557e769aa00 con 0x5557e7519800 2019-12-07T19:33:56.623+0000 7f2b95fb3700 15 osd.6 19 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.cs4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.623+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.cs4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.cs4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.cs4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.624+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 7 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x5557e7ade480 con 0x5557e7519800 2019-12-07T19:33:56.624+0000 7f2b834f8700 20 osd.6 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7f2b834f8700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:56.624+0000 7f2b834f8700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:56.624+0000 7f2b834f8700 20 osd.6 19 _dispatch 0x5557e7ade480 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(4) info: 1.cs4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.624+0000 7f2b834f8700 3 osd.6 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.624+0000 7f2b834f8700 10 osd.6 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.070847719s 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 36.070847719s 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.070847719s 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.070847719s) 2019-12-07T19:33:56.624+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.2(4) activated and committed 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 14 ==== pg_info2(1.cs0 1.cs0 1.cs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x5557e7699500 con 0x5557e79cac00 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(3) info: 1.cs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(3) info: 1.cs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(3) info: 1.cs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.624+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 15 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x5557e7adeb40 con 0x5557e79cac00 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.624+0000 7f2b834f8700 20 osd.6 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7f2b834f8700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:56.624+0000 7f2b834f8700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:56.624+0000 7f2b834f8700 20 osd.6 19 _dispatch 0x5557e7adeb40 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.624+0000 7f2b834f8700 3 osd.6 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.624+0000 7f2b834f8700 10 osd.6 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.624+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:4 2019-12-07T19:33:56.624+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(3) info: 1.cs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.624+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.624+0000 7f2b744da700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(3) info: 1.cs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.070847719s 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 36.070847719s 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 36.070847719s 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.070847719s 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.070847719s) 2019-12-07T19:33:56.624+0000 7f2b744da700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.1(3) activated and committed 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.624+0000 7f2b744da700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:5 2019-12-07T19:33:56.624+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.625+0000 7f2b9171a700 10 osd.6 19 tick_without_osd_lock 2019-12-07T19:33:56.625+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:56.625+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954630+0000 last_rx_front 2019-12-07T19:33:55.954652+0000 2019-12-07T19:33:56.625+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954509+0000 last_rx_front 2019-12-07T19:33:55.954559+0000 2019-12-07T19:33:56.625+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954547+0000 last_rx_front 2019-12-07T19:33:55.954615+0000 2019-12-07T19:33:56.625+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954643+0000 last_rx_front 2019-12-07T19:33:55.954592+0000 2019-12-07T19:33:56.625+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954574+0000 last_rx_front 2019-12-07T19:33:55.954605+0000 2019-12-07T19:33:56.625+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954662+0000 last_rx_front 2019-12-07T19:33:55.954672+0000 2019-12-07T19:33:56.625+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:55.954243+0000 last_rx_back 2019-12-07T19:33:55.954528+0000 last_rx_front 2019-12-07T19:33:55.954756+0000 2019-12-07T19:33:56.625+0000 7f2b9171a700 20 osd.6 19 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:56.625+0000 7f2b9171a700 20 osd.6 19 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:56.625+0000 7f2b9171a700 20 osd.6 19 scrub_load_below_threshold loadavg per cpu 0.05625 < max 5 = yes 2019-12-07T19:33:56.625+0000 7f2b9171a700 20 osd.6 19 sched_scrub load_is_low=1 2019-12-07T19:33:56.625+0000 7f2b9171a700 10 osd.6 19 sched_scrub 1.cs0 scheduled at 2019-12-07T19:35:16.655986+0000 > 2019-12-07T19:33:56.626274+0000 2019-12-07T19:33:56.625+0000 7f2b9171a700 20 osd.6 19 sched_scrub done 2019-12-07T19:33:56.625+0000 7f2b9171a700 10 osd.6 19 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:56.625+0000 7f2b9171a700 20 osd.6 19 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:56.625+0000 7f2b9171a700 10 osd.6 19 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:56.626+0000 7f2b92fad700 10 osd.6 19 tick 2019-12-07T19:33:56.626+0000 7f2b92fad700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:56.626+0000 7f2b92fad700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:56.626+0000 7f2b92fad700 20 osd.6 19 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:56.627+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 16 ==== pg_info2(1.7s1 1.7s1 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e70b6a00 con 0x5557e79cac00 2019-12-07T19:33:56.627+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.7s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.627+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) pg 0x5557e76bd000 2019-12-07T19:33:56.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.627+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:33:56.627+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7664300 seq 3 2019-12-07T19:33:56.627+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:56.627+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:56.627+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664300 onodes shared_blobs 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.627+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 prepare 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664300 2019-12-07T19:33:56.627+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 io_done 2019-12-07T19:33:56.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x18c22~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x18c22 0x18c22~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x8c22 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xc22 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x38 and padding block with 0xfc8 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x19038 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.627+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000098297s (0.000000078s flush + 0.000098219s kv commit) 2019-12-07T19:33:56.627+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7664300> 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.628+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 kv_submitted 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664300 2019-12-07T19:33:56.628+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 finishing 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664300 onodes 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664300 done 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:56.628+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664300 [] 2019-12-07T19:33:56.628+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 17 ==== pg_info2(1.7s1 1.7s1 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e70b6a00 con 0x5557e79cac00 2019-12-07T19:33:56.628+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.7s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.628+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) pg 0x5557e76bd000 2019-12-07T19:33:56.628+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.628+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:56.628+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7a92c00 seq 4 2019-12-07T19:33:56.628+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._biginfo' <- _biginfo 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:56.628+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:56.628+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes shared_blobs 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.628+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:56.628+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x19038~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x19038 0x19038~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x9038 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x38 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x485 and padding block with 0xb7b 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x19485 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.628+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000099413s (0.000000098s flush + 0.000099315s kv commit) 2019-12-07T19:33:56.628+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92c00> 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.628+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:56.628+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:56.628+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:56.628+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.628+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.628+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.639+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.689+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.703+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 8 ==== pg_info2(1.5s1 1.5s1 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e70b7800 con 0x5557e7519800 2019-12-07T19:33:56.703+0000 7f2b95fb3700 15 osd.6 19 enqueue_peering_evt 1.5s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s) 2019-12-07T19:33:56.703+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e19) pg 0x5557e764c000 2019-12-07T19:33:56.703+0000 7f2b73cd9700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s) 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.703+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 11 ==== pg_info2(1.6s1 1.6s1 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e70b6a00 con 0x5557e79ca800 2019-12-07T19:33:56.703+0000 7f2b73cd9700 10 log is not dirty 2019-12-07T19:33:56.703+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.6s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:33:56.703+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.703+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7630f00 seq 3 2019-12-07T19:33:56.703+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.703+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:56.703+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630f00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630f00 onodes shared_blobs 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.703+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 prepare 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630f00 2019-12-07T19:33:56.703+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 io_done 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x19485~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x19485 0x19485~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x9485 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x485 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x89b and padding block with 0x765 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1989b 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000102452s (0.000000157s flush + 0.000102295s kv commit) 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630f00> 2019-12-07T19:33:56.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.703+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 kv_submitted 2019-12-07T19:33:56.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630f00 2019-12-07T19:33:56.703+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 finishing 2019-12-07T19:33:56.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630f00 onodes 2019-12-07T19:33:56.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630f00 done 2019-12-07T19:33:56.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:56.703+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630f00 [] 2019-12-07T19:33:56.703+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.703+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.703+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 12 ==== pg_info2(1.6s1 1.6s1 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e70b6a00 con 0x5557e79ca800 2019-12-07T19:33:56.703+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.6s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:33:56.703+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.703+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 9 ==== pg_info2(1.5s1 1.5s1 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e70b7800 con 0x5557e7519800 2019-12-07T19:33:56.703+0000 7f2b95fb3700 15 osd.6 19 enqueue_peering_evt 1.5s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s) 2019-12-07T19:33:56.703+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e19) pg 0x5557e764c000 2019-12-07T19:33:56.703+0000 7f2b77ce1700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.5s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s) 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.703+0000 7f2b77ce1700 10 log is not dirty 2019-12-07T19:33:56.703+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7692600 seq 4 2019-12-07T19:33:56.703+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.703+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:56.703+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692600 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692600 onodes shared_blobs 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692600 allocated 0x[] released 0x[] 2019-12-07T19:33:56.703+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 prepare 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692600 2019-12-07T19:33:56.703+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 io_done 2019-12-07T19:33:56.703+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1989b~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1989b 0x1989b~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x989b 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x89b 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xce8 and padding block with 0x318 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x19ce8 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000087470s (0.000000098s flush + 0.000087372s kv commit) 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.704+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.704+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692600> 2019-12-07T19:33:56.704+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.704+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 kv_submitted 2019-12-07T19:33:56.704+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692600 2019-12-07T19:33:56.704+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 finishing 2019-12-07T19:33:56.704+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692600 onodes 2019-12-07T19:33:56.704+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692600 done 2019-12-07T19:33:56.704+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:56.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.704+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692600 [] 2019-12-07T19:33:56.704+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.704+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.704+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.704+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 3 ==== pg_info2(1.cs0 1.cs0 1.cs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x5557e70b6a00 con 0x5557e79ca000 2019-12-07T19:33:56.704+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(1) info: 1.cs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.704+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(1) info: 1.cs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(1) info: 1.cs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(1) info: 1.cs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s)) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(1) info: 1.cs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.070847719s) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 36.070847719s 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.070847719s 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 36.070847719s 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.2 ruub 36.070847719s 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.070847719s 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 36.070847719s (sent 36.070847719s) 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.5(1) activated and committed 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.704+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:6 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: all_activated_and_committed 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] state: mnow 20.160350345s >= prior_readable_until_ub 0s 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 19 send_pg_created 2019-12-07T19:33:56.704+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- osd_pg_created(1.c) v1 -- 0x5557e75b45a0 con 0x5557e6f87000 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] share_pg_info 2019-12-07T19:33:56.704+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e769b800 con 0x5557e79cac00 2019-12-07T19:33:56.704+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7698000 con 0x5557e7519800 2019-12-07T19:33:56.704+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7698e00 con 0x5557e79ca800 2019-12-07T19:33:56.704+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7699500 con 0x5557e79cbc00 2019-12-07T19:33:56.704+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x5557e7698700 con 0x5557e79ca000 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.704+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 13 ==== pg_info2(1.1s2 1.1s2 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e70b6a00 con 0x5557e79ca800 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.704+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.1s2 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.704+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.704+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:7 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] check_local 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) pg 0x5557e769f000 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_backfill does not need backfill 2019-12-07T19:33:56.704+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:33:56.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] activate all replicas clean, no recovery 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.704+0000 7f2b784e2700 15 osd.6 19 enqueue_peering_evt 1.cs0 epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) 2019-12-07T19:33:56.704+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:56.704+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e7631800 seq 3 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.704+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:56.704+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631800 onodes shared_blobs 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 prepare 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631800 2019-12-07T19:33:56.704+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 io_done 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.704+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x19ce8~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.704+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x19ce8 0x19ce8~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x9ce8 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xce8 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.704+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xfe and padding block with 0xf02 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.704+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:8 2019-12-07T19:33:56.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] agent_stop 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7631200 seq 4 2019-12-07T19:33:56.705+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 18 ==== pg_info2(1.es5 1.es5 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e7698700 con 0x5557e79cac00 2019-12-07T19:33:56.705+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631200 cost 5258 (1 ios * 4000 + 1258 bytes) 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631200 onodes shared_blobs 2019-12-07T19:33:56.705+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.es5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.705+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 prepare 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631200 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 io_done 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 19 ==== pg_info2(1.9s5 1.9s5 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e769b800 con 0x5557e79cac00 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) pg 0x5557e6e89000 2019-12-07T19:33:56.705+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.9s5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.705+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered 2019-12-07T19:33:56.705+0000 7f2b784e2700 5 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] exit Started/Primary/Active/Activating 2019-12-07T19:33:56.706276+0000 9 0.002194 2019-12-07T19:33:56.705+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 20 ==== pg_info2(1.as1 1.as1 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e769b800 con 0x5557e79cac00 2019-12-07T19:33:56.705+0000 7f2b784e2700 5 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] enter Started/Primary/Active/Recovered 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.705+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.as1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:33:56.705+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 1(3),2(4),3(5),4(2),5(1),6(0) upset 1(3),2(4),3(5),4(2),5(1),6(0) acting_recovery_backfill 1(3),2(4),3(5),4(2),5(1),6(0) 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 6(0) primary objects 0 missing 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(3) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 2(4) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) pg 0x5557e767c000 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(2) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(1) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(3) missing= 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 2(4) missing= 0 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:56.705+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 4 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x5557e7ade900 con 0x5557e79ca000 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7692900 seq 3 2019-12-07T19:33:56.705+0000 7f2b834f8700 20 osd.6 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.705+0000 7f2b834f8700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:56.705+0000 7f2b834f8700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:56.705+0000 7f2b834f8700 20 osd.6 19 _dispatch 0x5557e7ade900 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1a0fe 2019-12-07T19:33:56.705+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:56.705+0000 7f2b834f8700 3 osd.6 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.705+0000 7f2b834f8700 10 osd.6 19 no new maps here, dropping 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.705+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692900 onodes shared_blobs 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000256811s (0.000000104s flush + 0.000256707s kv commit) 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7631800> 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 kv_submitted 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631800 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 prepare 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692900 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 io_done 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1089 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 finishing 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631800 onodes 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631800 done 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631800 [] 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1112 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1112 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1a0fe~458 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1a0fe 0x1a0fe~458 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xa0fe 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(2) missing= 0 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xfe 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x556 and padding block with 0xaaa 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(1) missing= 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 6(0) missing= 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.705+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:9 2019-12-07T19:33:56.705+0000 7f2b784e2700 5 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] exit Started/Primary/Active/Recovered 2019-12-07T19:33:56.706474+0000 0 0.000000 2019-12-07T19:33:56.705+0000 7f2b784e2700 5 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] enter Started/Primary/Active/Clean 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] share_pg_info 2019-12-07T19:33:56.705+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x5557e9b12000 con 0x5557e79cac00 2019-12-07T19:33:56.705+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x5557e9b12700 con 0x5557e7519800 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1a556 2019-12-07T19:33:56.705+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x5557e9b12e00 con 0x5557e79ca800 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.705+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x5557e9b13500 con 0x5557e79cbc00 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000149155s (0.000000068s flush + 0.000149087s kv commit) 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x5557e9b13c00 con 0x5557e79ca000 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1a556~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1a556 0x1a556~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:10 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xa556 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x556 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x96c and padding block with 0x694 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_recovery 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] clear_recovery_state 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7631200> 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 kv_submitted 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631200 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 finishing 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631200 onodes 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631200 done 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631200 [] 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7692c00 seq 5 2019-12-07T19:33:56.705+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692c00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692c00 onodes shared_blobs 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 prepare 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692c00 2019-12-07T19:33:56.705+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 io_done 2019-12-07T19:33:56.705+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 4 ==== pg_info2(1.2s3 1.2s3 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e7699c00 con 0x5557e79cb800 2019-12-07T19:33:56.705+0000 7f2b967b4700 15 osd.6 19 enqueue_peering_evt 1.2s3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s) 2019-12-07T19:33:56.705+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) pg 0x5557e766d000 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e7693500 seq 3 2019-12-07T19:33:56.705+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693500 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693500 onodes shared_blobs 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1a96c 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 prepare 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693500 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 io_done 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 14 ==== pg_info2(1.1s2 1.1s2 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79ca800 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e19) pg 0x5557e767f000 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000286698s (0.000000093s flush + 0.000286605s kv commit) 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.705+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.1s2 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s) 2019-12-07T19:33:56.705+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 1 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e7693800 seq 3 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1a96c~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1a96c 0x1a96c~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693800 onodes shared_blobs 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xa96c 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x96c 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x1b8 and padding block with 0xe48 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 prepare 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693800 2019-12-07T19:33:56.705+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 io_done 2019-12-07T19:33:56.705+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) pg 0x5557e769f000 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692900> 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 kv_submitted 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692900 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 finishing 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692900 onodes 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 done 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692900 [] 2019-12-07T19:33:56.705+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.705+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e9b10000 seq 4 2019-12-07T19:33:56.705+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._biginfo' <- _biginfo 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10000 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10000 onodes shared_blobs 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 prepare 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10000 2019-12-07T19:33:56.705+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 io_done 2019-12-07T19:33:56.705+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 5 ==== pg_info2(1.2s3 1.2s3 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e7699c00 con 0x5557e79cb800 2019-12-07T19:33:56.705+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7f2b967b4700 15 osd.6 19 enqueue_peering_evt 1.2s3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s) 2019-12-07T19:33:56.705+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.705+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.705+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e19) pg 0x5557e767f000 2019-12-07T19:33:56.705+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 0(0) info: 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s) 2019-12-07T19:33:56.705+0000 7f2b794e4700 20 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.705+0000 7f2b794e4700 20 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.705+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1b1b8 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.705+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:56.705+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.705+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.705+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e9b10600 seq 4 2019-12-07T19:33:56.706+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000347003s (0.000000083s flush + 0.000346920s kv commit) 2019-12-07T19:33:56.706+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.706+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:56.706+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10600 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:56.706+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10600 onodes shared_blobs 2019-12-07T19:33:56.706+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10600 allocated 0x[] released 0x[] 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1b1b8~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10600 prepare 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1b1b8 0x1b1b8~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xb1b8 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x1b8 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa04 and padding block with 0x5fc 2019-12-07T19:33:56.706+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10600 2019-12-07T19:33:56.706+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10600 io_done 2019-12-07T19:33:56.706+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.706+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692c00,0x5557e7693500> 2019-12-07T19:33:56.706+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.706+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 kv_submitted 2019-12-07T19:33:56.706+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692c00 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1ba04 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000173583s (0.000000071s flush + 0.000173512s kv commit) 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1ba04~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1ba04 0x1ba04~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xba04 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa04 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe51 and padding block with 0x1af 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1be51 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.706+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000184360s (0.000000063s flush + 0.000184297s kv commit) 2019-12-07T19:33:56.706+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.705+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 finishing 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692c00 onodes 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692c00 done 2019-12-07T19:33:56.707+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692c00 [] 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 kv_submitted 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693500 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 finishing 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693500 onodes 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693500 done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9e00 q now empty 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693500 [] 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693800,0x5557e9b10000,0x5557e9b10600> 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 kv_submitted 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693800 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 finishing 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693800 onodes 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693800 done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10600 kv_submitted 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693800 [] 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 kv_submitted 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10000 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 finishing 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10000 onodes 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10000 done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10000 [] 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10600 kv_submitted 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10600 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10600 finishing 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10600 onodes 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10600 done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10600 [] 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] _finish_recovery 2019-12-07T19:33:56.707+0000 7f2b784e2700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] purge_strays 2019-12-07T19:33:56.707+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.707+0000 7f2b784e2700 15 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:11 2019-12-07T19:33:56.707+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 10 ==== pg_info2(1.bs1 1.bs1 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e9b12e00 con 0x5557e79cbc00 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.bs1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 11 ==== pg_info2(1.3s5 1.3s5 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e9b12e00 con 0x5557e79cbc00 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.3s5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 12 ==== pg_info2(1.bs1 1.bs1 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e9b12e00 con 0x5557e79cbc00 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.bs1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 13 ==== pg_info2(1.3s5 1.3s5 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e9b12e00 con 0x5557e79cbc00 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.3s5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 21 ==== pg_info2(1.8s5 1.8s5 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e9b12e00 con 0x5557e79cac00 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.8s5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 22 ==== pg_info2(1.es5 1.es5 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e9b12000 con 0x5557e79cac00 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e19) pg 0x5557e76ba000 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.es5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b764de700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s) 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 23 ==== pg_info2(1.8s5 1.8s5 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e9b12000 con 0x5557e79cac00 2019-12-07T19:33:56.707+0000 7f2b764de700 10 log is not dirty 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.8s5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7664300 seq 3 2019-12-07T19:33:56.707+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 24 ==== pg_info2(1.9s5 1.9s5 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e9b12000 con 0x5557e79cac00 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.9s5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664300 onodes shared_blobs 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 25 ==== pg_info2(1.as1 1.as1 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x5557e9b12000 con 0x5557e79cac00 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 prepare 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664300 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 io_done 2019-12-07T19:33:56.707+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.as1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:33:56.707+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e19) pg 0x5557e76ba000 2019-12-07T19:33:56.707+0000 7f2b764de700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.8s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s) 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.707+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) pg 0x5557e7957000 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.707+0000 7f2b764de700 10 log is not dirty 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) pg 0x5557e767c000 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:56.707+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7630f00 seq 4 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:33:56.707+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._biginfo' <- _biginfo 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630f00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630f00 onodes shared_blobs 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7631800 seq 3 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.707+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:33:56.707+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 prepare 2019-12-07T19:33:56.707+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630f00 2019-12-07T19:33:56.707+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 io_done 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.707+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e9b10300 seq 4 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631800 onodes shared_blobs 2019-12-07T19:33:56.707+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 prepare 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631800 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 io_done 2019-12-07T19:33:56.707+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10300 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10300 onodes shared_blobs 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 2 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:56.707+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 prepare 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) pg 0x5557e7954000 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10300 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 io_done 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1be51~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1be51 0x1be51~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xbe51 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe51 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x69d and padding block with 0x963 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7692600 seq 3 2019-12-07T19:33:56.707+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692600 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692600 onodes shared_blobs 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692600 allocated 0x[] released 0x[] 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 prepare 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692600 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 io_done 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) pg 0x5557e7957000 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) pg 0x5557e766d000 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:33:56.707+0000 7f2b754dc700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7631200 seq 4 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.707+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631200 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.707+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631200 onodes shared_blobs 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.707+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 prepare 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e7693b00 seq 4 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631200 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 io_done 2019-12-07T19:33:56.707+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e19) pg 0x5557e7954000 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.707+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:56.707+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693b00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693b00 onodes shared_blobs 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.3s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.707+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 prepare 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693b00 2019-12-07T19:33:56.707+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 io_done 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.707+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7692900 seq 4 2019-12-07T19:33:56.707+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1c69d 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692900 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692900 onodes shared_blobs 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000195234s (0.000000100s flush + 0.000195134s kv commit) 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 prepare 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692900 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:33:56.707+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 io_done 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 2 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3124 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 2 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4202 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5280 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 1 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5303 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5303 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1c69d~14b7 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1c69d 0x1c69d~14b7 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xc69d 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x69d 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.707+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xb54 and padding block with 0x4ac 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7664300,0x5557e7630f00> 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 kv_submitted 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664300 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664300 finishing 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664300 onodes 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664300 done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630f00 kv_submitted 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664300 [] 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 kv_submitted 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630f00 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 finishing 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630f00 onodes 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630f00 done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630f00 [] 2019-12-07T19:33:56.707+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.707+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) pg 0x5557e769c000 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9b10900 seq 3 2019-12-07T19:33:56.708+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10900 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 prepare 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10900 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 io_done 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e19) pg 0x5557e769c000 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.6s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9b10c00 seq 4 2019-12-07T19:33:56.708+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10c00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10c00 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10c00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 prepare 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10c00 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 io_done 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) pg 0x5557e766a000 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e9b10f00 seq 3 2019-12-07T19:33:56.708+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10f00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10f00 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 prepare 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 io_done 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) pg 0x5557e766a000 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e9b11200 seq 4 2019-12-07T19:33:56.708+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1db54 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11200 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11200 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 5 cleaned 0 in 0.000589483s (0.000000090s flush + 0.000589393s kv commit) 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7631800,0x5557e9b10300,0x5557e7692600,0x5557e7631200,0x5557e7693b00> 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 prepare 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11200 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631800 2019-12-07T19:33:56.708+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 io_done 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631800 finishing 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631800 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631800 done 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631200 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631800 [] 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10300 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 2 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 finishing 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10300 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10300 done 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3179 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10300 [] 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692600 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 finishing 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692600 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692600 done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692600 [] 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4202 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631200 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 finishing 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 2 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631200 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631200 done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631200 [] 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4225 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4225 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693b00 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 finishing 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693b00 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693b00 done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9e00 q now empty 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693b00 [] 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1db54~1081 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1db54 0x1db54~1081 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xdb54 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xb54 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.708+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xbd5 and padding block with 0x42b 2019-12-07T19:33:56.708+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1ebd5 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 4 cleaned 0 in 0.000130340s (0.000000049s flush + 0.000130291s kv commit) 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1ebd5~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1ebd5 0x1ebd5~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xebd5 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xbd5 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x22 and padding block with 0xfde 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692900,0x5557e9b10900,0x5557e9b10c00,0x5557e9b10f00> 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692900 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 finishing 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692900 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692900 [] 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10900 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 finishing 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10900 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10900 done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10900 [] 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10c00 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 finishing 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10c00 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10c00 [] 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10f00 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 finishing 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10f00 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11200 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10f00 [] 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.708+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1f022 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.708+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000147390s (0.000000064s flush + 0.000147326s kv commit) 2019-12-07T19:33:56.708+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11200> 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 kv_submitted 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11200 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 finishing 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11200 onodes 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11200 done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11200 [] 2019-12-07T19:33:56.708+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.708+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.708+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 15 ==== pg_info2(1.fs5 1.fs5 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e9b12000 con 0x5557e79ca800 2019-12-07T19:33:56.710+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.fs5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:33:56.710+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) pg 0x5557e764f000 2019-12-07T19:33:56.710+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.710+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:33:56.710+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7664900 seq 3 2019-12-07T19:33:56.710+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.710+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:56.710+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664900 onodes shared_blobs 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.710+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 prepare 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664900 2019-12-07T19:33:56.710+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 io_done 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1f022~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1f022 0x1f022~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xf022 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x22 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x438 and padding block with 0xbc8 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1f438 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.710+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000085117s (0.000000079s flush + 0.000085038s kv commit) 2019-12-07T19:33:56.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7664900> 2019-12-07T19:33:56.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 kv_submitted 2019-12-07T19:33:56.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664900 2019-12-07T19:33:56.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 finishing 2019-12-07T19:33:56.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664900 onodes 2019-12-07T19:33:56.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664900 done 2019-12-07T19:33:56.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:56.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664900 [] 2019-12-07T19:33:56.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 16 ==== pg_info2(1.fs5 1.fs5 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (crc 0 0 0) 0x5557e9b12000 con 0x5557e79ca800 2019-12-07T19:33:56.710+0000 7f2b957b2700 15 osd.6 19 enqueue_peering_evt 1.fs5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:33:56.710+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) pg 0x5557e764f000 2019-12-07T19:33:56.710+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:33:56.710+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.711+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:56.711+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7046f00 seq 4 2019-12-07T19:33:56.711+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.711+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:56.711+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes shared_blobs 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.711+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:56.711+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1f438~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1f438 0x1f438~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xf438 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x438 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x885 and padding block with 0x77b 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x1f885 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:56.711+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000102409s (0.000000040s flush + 0.000102369s kv commit) 2019-12-07T19:33:56.711+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:56.711+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:56.711+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:56.711+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.711+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:56.711+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:56.711+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:56.711+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 2019-12-07T19:33:56.711+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:56.711+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:56.711+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:56.711+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:56.711+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:56.711+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.711+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.739+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.789+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.839+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.889+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.940+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.989+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.040+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.053+0000 7f2b724d6700 5 osd.6 19 heartbeat osd_stat(store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [0,1,2,3,4,5,7] op hist []) 2019-12-07T19:33:57.053+0000 7f2b724d6700 20 osd.6 19 check_full_status cur ratio 2.07689e-06, physical ratio 2.07689e-06, new state none 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.000099360s) v5 -- 0x5557e7617c00 con 0x5557e7a49000 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.000099360s) v5 -- 0x5557e7617e00 con 0x5557e7a49400 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 18446744073.707155507s) v5 -- 0x5557e9b36000 con 0x5557e7af0800 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 18446744073.707155507s) v5 -- 0x5557e9b36200 con 0x5557e7af0c00 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 18446744073.692423660s) v5 -- 0x5557e9b36400 con 0x5557e7af1000 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 18446744073.692423660s) v5 -- 0x5557e9b36600 con 0x5557e7af1400 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 18446744073.706611476s) v5 -- 0x5557e9b36800 con 0x5557e7af1800 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 18446744073.706611476s) v5 -- 0x5557e9b36a00 con 0x5557e7af1c00 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.005405600s) v5 -- 0x5557e9b36c00 con 0x5557e75aa000 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.005405600s) v5 -- 0x5557e9b36e00 con 0x5557e75aa400 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 18446744073.697991432s) v5 -- 0x5557e9b37000 con 0x5557e7a49800 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 18446744073.697991432s) v5 -- 0x5557e9b37200 con 0x5557e7a49c00 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.019106793s) v5 -- 0x5557e9b37400 con 0x5557e7af0000 2019-12-07T19:33:57.053+0000 7f2b724d6700 1 -- 172.21.15.109:0/36937 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.019106793s) v5 -- 0x5557e9b37600 con 0x5557e7af0400 2019-12-07T19:33:57.053+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6830/37079 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509657308s delta_ub 0.000499460s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9b37000 con 0x5557e7a49000 2019-12-07T19:33:57.053+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6827/37078 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.511906108s delta_ub 0.002748260s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af0c00 2019-12-07T19:33:57.053+0000 7f2b95fb3700 25 osd.6 19 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954630+0000 -> 2019-12-07T19:33:57.055010+0000 last_rx_front 2019-12-07T19:33:55.954652+0000 2019-12-07T19:33:57.053+0000 7f2b95fb3700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.000141411s,0.000499460s]) 2019-12-07T19:33:57.053+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6806/37077 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.526727432s delta_ub 0.017569584s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7af1400 2019-12-07T19:33:57.053+0000 7f2b957b2700 25 osd.6 19 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954547+0000 last_rx_front 2019-12-07T19:33:55.954615+0000 -> 2019-12-07T19:33:57.055052+0000 2019-12-07T19:33:57.053+0000 7f2b957b2700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017169819s,0.017569584s]) 2019-12-07T19:33:57.053+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6806/36936 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.521305977s delta_ub 0.012148129s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9b37000 con 0x5557e7a49800 2019-12-07T19:33:57.053+0000 7f2b95fb3700 25 osd.6 19 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954662+0000 -> 2019-12-07T19:33:57.055064+0000 last_rx_front 2019-12-07T19:33:55.954672+0000 2019-12-07T19:33:57.053+0000 7f2b95fb3700 10 osd.6 19 maybe_share_map con 0x5557e79ca000 v2:172.21.15.109:6802/36936 map epoch 18 -> 19 (as per caller) 2019-12-07T19:33:57.053+0000 7f2b95fb3700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011735583s,0.012148129s]) 2019-12-07T19:33:57.053+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.0 v2:172.21.15.131:6826/37079 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509735039s delta_ub 0.000577191s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7a49400 2019-12-07T19:33:57.053+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.2 v2:172.21.15.131:6808/37077 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.526771556s delta_ub 0.017613708s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9b37000 con 0x5557e7af1000 2019-12-07T19:33:57.053+0000 7f2b957b2700 25 osd.6 19 handle_osd_ping got reply from osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055010+0000 last_rx_front 2019-12-07T19:33:55.954652+0000 -> 2019-12-07T19:33:57.055094+0000 2019-12-07T19:33:57.053+0000 7f2b957b2700 25 osd.6 19 handle_osd_ping got all replies from osd.0 , erase pending ping(sent at 2019-12-07T19:33:57.054652+0000) and older pending ping(s) 2019-12-07T19:33:57.053+0000 7f2b957b2700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.0 up_from 13 peer_clock_delta [0.000135583s,0.000577191s]) 2019-12-07T19:33:57.053+0000 7f2b95fb3700 25 osd.6 19 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954547+0000 -> 2019-12-07T19:33:57.055116+0000 last_rx_front 2019-12-07T19:33:57.055052+0000 2019-12-07T19:33:57.053+0000 7f2b95fb3700 25 osd.6 19 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:57.054652+0000) and older pending ping(s) 2019-12-07T19:33:57.053+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6814/36938 2 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.504235004s delta_ub 18446744073.704628772s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e75aa000 2019-12-07T19:33:57.053+0000 7f2b95fb3700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017149841s,0.017613708s]) 2019-12-07T19:33:57.053+0000 7f2b967b4700 25 osd.6 19 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954509+0000 last_rx_front 2019-12-07T19:33:55.954559+0000 -> 2019-12-07T19:33:57.055135+0000 2019-12-07T19:33:57.053+0000 7f2b967b4700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002265041s,0.002748260s]) 2019-12-07T19:33:57.053+0000 7f2b957b2700 25 osd.6 19 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954574+0000 -> 2019-12-07T19:33:57.055145+0000 last_rx_front 2019-12-07T19:33:55.954605+0000 2019-12-07T19:33:57.053+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.4 v2:172.21.15.109:6812/36938 2 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.504202645s delta_ub 18446744073.704596413s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e75aa400 2019-12-07T19:33:57.053+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6828/36939 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.490487301s delta_ub 18446744073.690881069s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9b37000 con 0x5557e7af0400 2019-12-07T19:33:57.053+0000 7f2b957b2700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704135544s,18446744073.704628772s]) 2019-12-07T19:33:57.053+0000 7f2b95fb3700 25 osd.6 19 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954528+0000 last_rx_front 2019-12-07T19:33:55.954756+0000 -> 2019-12-07T19:33:57.055160+0000 2019-12-07T19:33:57.053+0000 7f2b95fb3700 10 osd.6 19 maybe_share_map con 0x5557e79f4000 v2:172.21.15.109:6826/36939 map epoch 18 -> 19 (as per caller) 2019-12-07T19:33:57.053+0000 7f2b95fb3700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690373374s,18446744073.690881069s]) 2019-12-07T19:33:57.053+0000 7f2b967b4700 25 osd.6 19 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055145+0000 last_rx_front 2019-12-07T19:33:55.954605+0000 -> 2019-12-07T19:33:57.055175+0000 2019-12-07T19:33:57.053+0000 7f2b967b4700 25 osd.6 19 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:57.054652+0000) and older pending ping(s) 2019-12-07T19:33:57.053+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.5 v2:172.21.15.109:6804/36936 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.521330478s delta_ub 0.012172630s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7a49c00 2019-12-07T19:33:57.053+0000 7f2b967b4700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704073467s,18446744073.704596413s]) 2019-12-07T19:33:57.053+0000 7f2b95fb3700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6814/37076 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.512616838s delta_ub 0.003458990s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9b37000 con 0x5557e7af1c00 2019-12-07T19:33:57.053+0000 7f2b957b2700 25 osd.6 19 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055064+0000 last_rx_front 2019-12-07T19:33:55.954672+0000 -> 2019-12-07T19:33:57.055188+0000 2019-12-07T19:33:57.053+0000 7f2b957b2700 25 osd.6 19 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:57.054652+0000) and older pending ping(s) 2019-12-07T19:33:57.054+0000 7f2b957b2700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011636594s,0.012172630s]) 2019-12-07T19:33:57.053+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.3 v2:172.21.15.131:6816/37076 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.512561942s delta_ub 0.003404094s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af1800 2019-12-07T19:33:57.054+0000 7f2b967b4700 25 osd.6 19 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954643+0000 -> 2019-12-07T19:33:57.055209+0000 last_rx_front 2019-12-07T19:33:55.954592+0000 2019-12-07T19:33:57.054+0000 7f2b967b4700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002846617s,0.003404094s]) 2019-12-07T19:33:57.054+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6831/37078 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.512018084s delta_ub 0.002860236s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e7af0800 2019-12-07T19:33:57.054+0000 7f2b95fb3700 25 osd.6 19 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055209+0000 last_rx_front 2019-12-07T19:33:55.954592+0000 -> 2019-12-07T19:33:57.055220+0000 2019-12-07T19:33:57.054+0000 7f2b967b4700 1 -- 172.21.15.109:0/36937 <== osd.7 v2:172.21.15.109:6830/36939 5 ==== osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.490455463s delta_ub 18446744073.690849231s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e7af0000 2019-12-07T19:33:57.054+0000 7f2b95fb3700 25 osd.6 19 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:57.054652+0000) and older pending ping(s) 2019-12-07T19:33:57.054+0000 7f2b95fb3700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002890878s,0.003458990s]) 2019-12-07T19:33:57.054+0000 7f2b957b2700 25 osd.6 19 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954509+0000 -> 2019-12-07T19:33:57.055240+0000 last_rx_front 2019-12-07T19:33:57.055135+0000 2019-12-07T19:33:57.054+0000 7f2b957b2700 25 osd.6 19 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:57.054652+0000) and older pending ping(s) 2019-12-07T19:33:57.054+0000 7f2b957b2700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002272164s,0.002860236s]) 2019-12-07T19:33:57.054+0000 7f2b967b4700 25 osd.6 19 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:55.954528+0000 -> 2019-12-07T19:33:57.055254+0000 last_rx_front 2019-12-07T19:33:57.055160+0000 2019-12-07T19:33:57.054+0000 7f2b967b4700 25 osd.6 19 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:57.054652+0000) and older pending ping(s) 2019-12-07T19:33:57.054+0000 7f2b967b4700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690247529s,18446744073.690849231s]) 2019-12-07T19:33:57.071+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.7 172.21.15.109:0/36939 5 ==== osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.690881069s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e748f800 2019-12-07T19:33:57.071+0000 7f2b957b2700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690325900s,18446744073.690881069s]) 2019-12-07T19:33:57.071+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.527128116s delta_ub 0.019225716s) v5 -- 0x5557e9b37200 con 0x5557e748f800 2019-12-07T19:33:57.071+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.7 172.21.15.109:0/36939 5 ==== osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.690881069s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758cc00 con 0x5557e74cc800 2019-12-07T19:33:57.071+0000 7f2b957b2700 20 osd.6 19 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690174194s,18446744073.690881069s]) 2019-12-07T19:33:57.071+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.527279822s delta_ub 0.019377422s) v5 -- 0x5557e9b37200 con 0x5557e74cc800 2019-12-07T19:33:57.090+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.140+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.190+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.240+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.290+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.341+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.391+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.441+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.491+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.541+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.591+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:33:57.591+0000 7f2b8ad07700 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock osr 0x5557e76ce500 2 ios pending 2019-12-07T19:33:57.591+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock seq 9 0x2000~1000 2019-12-07T19:33:57.591+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock write 0x2000~1000 crc 1aeb1035 2019-12-07T19:33:57.591+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock seq 9 0x1b000~1000 2019-12-07T19:33:57.591+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock write 0x1b000~1000 crc 5f344e98 2019-12-07T19:33:57.591+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.591+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_aio_finish osr 0x5557e76ce500 2019-12-07T19:33:57.591+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_aio_finish dequeueing 2019-12-07T19:33:57.616+0000 7f2b9171a700 10 osd.6 19 tick_without_osd_lock 2019-12-07T19:33:57.616+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:57.616+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055010+0000 last_rx_front 2019-12-07T19:33:57.055094+0000 2019-12-07T19:33:57.616+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055240+0000 last_rx_front 2019-12-07T19:33:57.055135+0000 2019-12-07T19:33:57.616+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055116+0000 last_rx_front 2019-12-07T19:33:57.055052+0000 2019-12-07T19:33:57.616+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055209+0000 last_rx_front 2019-12-07T19:33:57.055220+0000 2019-12-07T19:33:57.616+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055145+0000 last_rx_front 2019-12-07T19:33:57.055175+0000 2019-12-07T19:33:57.616+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055064+0000 last_rx_front 2019-12-07T19:33:57.055188+0000 2019-12-07T19:33:57.616+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055254+0000 last_rx_front 2019-12-07T19:33:57.055160+0000 2019-12-07T19:33:57.616+0000 7f2b9171a700 20 osd.6 19 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:57.616+0000 7f2b9171a700 10 osd.6 19 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:57.616+0000 7f2b9171a700 20 osd.6 19 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:57.616+0000 7f2b9171a700 10 osd.6 19 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:57.628+0000 7f2b92fad700 10 osd.6 19 tick 2019-12-07T19:33:57.628+0000 7f2b92fad700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:57.628+0000 7f2b92fad700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:57.628+0000 7f2b92fad700 20 osd.6 19 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:57.641+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.691+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.742+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.791+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.842+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.892+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.942+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.992+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.042+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.092+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.143+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.193+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.243+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.251+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e6f87400 0x5557e7480680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.251+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e6f87400 0x5557e7480680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.251+0000 7f2b957b2700 10 osd.6 19 ms_handle_authentication new session 0x5557e7517ba0 con 0x5557e6f87400 entity client.admin addr 2019-12-07T19:33:58.251+0000 7f2b957b2700 10 osd.6 19 ms_handle_authentication session 0x5557e7517ba0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.252+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/4040951113 conn(0x5557e6f87400 0x5557e7480680 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4346 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:58.252+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4346 172.21.15.131:0/4040951113 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x5557e75e21e0 con 0x5557e6f87400 2019-12-07T19:33:58.252+0000 7f2b88017700 20 osd.6 19 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:58.252+0000 7f2b88017700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:58.252+0000 7f2b88017700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:58.252+0000 7f2b88017700 20 osd.6 19 _dispatch 0x5557e75e21e0 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:58.252+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/4040951113 -- command_reply(tid 13: 0 ) v1 -- 0x5557e627ee00 con 0x5557e6f87400 2019-12-07T19:33:58.277+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4346 172.21.15.131:0/4040951113 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x5557e75e2f00 con 0x5557e6f87400 2019-12-07T19:33:58.277+0000 7f2b88017700 20 osd.6 19 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:58.277+0000 7f2b88017700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:58.277+0000 7f2b88017700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:58.277+0000 7f2b88017700 20 osd.6 19 _dispatch 0x5557e75e2f00 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:58.278+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/4040951113 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x5557e627e380 con 0x5557e6f87400 2019-12-07T19:33:58.293+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.308+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/4040951113 conn(0x5557e6f87400 msgr2=0x5557e7480680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 95 2019-12-07T19:33:58.308+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/4040951113 conn(0x5557e6f87400 msgr2=0x5557e7480680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:58.308+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/4040951113 conn(0x5557e6f87400 0x5557e7480680 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.308+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/4040951113 conn(0x5557e6f87400 0x5557e7480680 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:58.308+0000 7f2b88017700 2 osd.6 19 ms_handle_reset con 0x5557e6f87400 session 0x5557e7517ba0 2019-12-07T19:33:58.343+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.393+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.443+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.493+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.544+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.594+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 67638 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.603+0000 7f2b9171a700 10 osd.6 19 tick_without_osd_lock 2019-12-07T19:33:58.603+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:58.603+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055010+0000 last_rx_front 2019-12-07T19:33:57.055094+0000 2019-12-07T19:33:58.603+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055240+0000 last_rx_front 2019-12-07T19:33:57.055135+0000 2019-12-07T19:33:58.603+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055116+0000 last_rx_front 2019-12-07T19:33:57.055052+0000 2019-12-07T19:33:58.603+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055209+0000 last_rx_front 2019-12-07T19:33:57.055220+0000 2019-12-07T19:33:58.603+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055145+0000 last_rx_front 2019-12-07T19:33:57.055175+0000 2019-12-07T19:33:58.603+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055064+0000 last_rx_front 2019-12-07T19:33:57.055188+0000 2019-12-07T19:33:58.603+0000 7f2b9171a700 25 osd.6 19 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055254+0000 last_rx_front 2019-12-07T19:33:57.055160+0000 2019-12-07T19:33:58.603+0000 7f2b9171a700 20 osd.6 19 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:58.603+0000 7f2b9171a700 20 osd.6 19 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:58.603+0000 7f2b9171a700 20 osd.6 19 scrub_load_below_threshold loadavg per cpu 0.05625 < max 5 = yes 2019-12-07T19:33:58.603+0000 7f2b9171a700 20 osd.6 19 sched_scrub load_is_low=1 2019-12-07T19:33:58.603+0000 7f2b9171a700 10 osd.6 19 sched_scrub 1.cs0 scheduled at 2019-12-07T19:35:16.655986+0000 > 2019-12-07T19:33:58.604366+0000 2019-12-07T19:33:58.603+0000 7f2b9171a700 20 osd.6 19 sched_scrub done 2019-12-07T19:33:58.603+0000 7f2b9171a700 10 osd.6 19 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:58.603+0000 7f2b9171a700 20 osd.6 19 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:58.603+0000 7f2b9171a700 10 osd.6 19 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:58.608+0000 7f2b92fad700 10 osd.6 19 tick 2019-12-07T19:33:58.608+0000 7f2b92fad700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:58.608+0000 7f2b92fad700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:58.608+0000 7f2b92fad700 20 osd.6 19 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:58.624+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e6f87800 0x5557e7a98100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.624+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e6f87800 0x5557e7a98100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.624+0000 7f2b957b2700 10 osd.6 19 ms_handle_authentication new session 0x5557e6295860 con 0x5557e6f87800 entity client.0 addr 2019-12-07T19:33:58.625+0000 7f2b957b2700 10 osd.6 19 ms_handle_authentication session 0x5557e6295860 client.0 has caps osdcap[grant(rwx)] 'allow rwx' 2019-12-07T19:33:58.625+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/224485236 conn(0x5557e6f87800 0x5557e7a98100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4540 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:58.625+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 1 ==== osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 ==== 263+0+32768 (crc 0 0 0) 0x5557e7a7ab00 con 0x5557e6f87800 2019-12-07T19:33:58.625+0000 7f2b957b2700 15 osd.6 19 enqueue_op 0x5557e76951e0 prio 63 cost 32768 latency 0.000049 epoch 20 osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.625+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e20) 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 2 ==== osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 ==== 225+0+62 (crc 0 0 0) 0x5557e7a7adc0 con 0x5557e6f87800 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e20) queued 2019-12-07T19:33:58.625+0000 7f2b957b2700 15 osd.6 19 enqueue_op 0x5557e76954a0 prio 63 cost 62 latency 0.000015 epoch 20 osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e20) pg 0x5557e6e89000 2019-12-07T19:33:58.625+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e20) 2019-12-07T19:33:58.625+0000 7f2b744da700 10 osd.6 19 dequeue_op 0x5557e76951e0 prio 63 cost 32768 latency 0.000109 osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 3 ==== osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 ==== 225+0+0 (crc 0 0 0) 0x5557e7a7b080 con 0x5557e6f87800 2019-12-07T19:33:58.625+0000 7f2b744da700 10 osd.6 19 maybe_share_map con 0x5557e6f87800 172.21.15.131:0/224485236 map epoch 0 -> 20 (as per caller) 2019-12-07T19:33:58.625+0000 7f2b957b2700 15 osd.6 19 enqueue_op 0x5557e7695760 prio 63 cost 0 latency 0.000009 epoch 20 osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e20) queued 2019-12-07T19:33:58.625+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e20) 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_request min 20, queue on waiting_for_map client.4540 2019-12-07T19:33:58.625+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.109:3300/0,v1:172.21.15.109:6789/0] -- mon_subscribe({osdmap=20}) v3 -- 0x5557e75a1c00 con 0x5557e6f87000 2019-12-07T19:33:58.625+0000 7f2b744da700 10 osd.6 19 dequeue_op 0x5557e76951e0 finish 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e20) queued 2019-12-07T19:33:58.625+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 4 ==== osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 ==== 263+0+32768 (crc 0 0 0) 0x5557e7a7b340 con 0x5557e6f87800 2019-12-07T19:33:58.625+0000 7f2b957b2700 15 osd.6 19 enqueue_op 0x5557e76958c0 prio 63 cost 32768 latency 0.000012 epoch 20 osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.625+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e20) 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e20) pg 0x5557e6e89000 2019-12-07T19:33:58.625+0000 7f2b784e2700 10 osd.6 19 dequeue_op 0x5557e76954a0 prio 63 cost 62 latency 0.000118 osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.625+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 5 ==== osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 ==== 225+0+62 (crc 0 0 0) 0x5557e7a7b600 con 0x5557e6f87800 2019-12-07T19:33:58.625+0000 7f2b957b2700 15 osd.6 19 enqueue_op 0x5557e7695b80 prio 63 cost 62 latency 0.000007 epoch 20 osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:33:58.625+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e20) 2019-12-07T19:33:58.625+0000 7f2b784e2700 10 osd.6 19 dequeue_op 0x5557e76954a0 finish 2019-12-07T19:33:58.625+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 6 ==== osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 ==== 225+0+0 (crc 0 0 0) 0x5557e7a7b8c0 con 0x5557e6f87800 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e20) queued 2019-12-07T19:33:58.625+0000 7f2b957b2700 15 osd.6 19 enqueue_op 0x5557e7695e40 prio 63 cost 0 latency 0.000008 epoch 20 osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.625+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e20) 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e20) pg 0x5557e6e89000 2019-12-07T19:33:58.625+0000 7f2b744da700 10 osd.6 19 dequeue_op 0x5557e7695760 prio 63 cost 0 latency 0.000142 osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 pg pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:33:58.625+0000 7f2b744da700 10 osd.6 19 dequeue_op 0x5557e7695760 finish 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e20) queued 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e20) pg 0x5557e6e89000 2019-12-07T19:33:58.625+0000 7f2b784e2700 10 osd.6 19 dequeue_op 0x5557e76958c0 prio 63 cost 32768 latency 0.000148 osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.625+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== mon.0 v2:172.21.15.109:3300/0 19 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (crc 0 0 0) 0x5557e7adf8c0 con 0x5557e6f87000 2019-12-07T19:33:58.625+0000 7f2b88017700 20 osd.6 19 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.625+0000 7f2b88017700 10 osd.6 19 do_waiters -- start 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:33:58.625+0000 7f2b784e2700 10 osd.6 19 dequeue_op 0x5557e76958c0 finish 2019-12-07T19:33:58.625+0000 7f2b88017700 10 osd.6 19 do_waiters -- finish 2019-12-07T19:33:58.625+0000 7f2b88017700 20 osd.6 19 _dispatch 0x5557e7adf8c0 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e20) queued 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e20) pg 0x5557e6e89000 2019-12-07T19:33:58.625+0000 7f2b88017700 3 osd.6 19 handle_osd_map epochs [20,21], i have 19, src has [1,21] 2019-12-07T19:33:58.625+0000 7f2b88017700 10 osd.6 19 handle_osd_map got inc map for epoch 20 2019-12-07T19:33:58.625+0000 7f2b744da700 10 osd.6 19 dequeue_op 0x5557e7695b80 prio 63 cost 62 latency 0.000163 osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:33:58.625+0000 7f2b744da700 10 osd.6 19 dequeue_op 0x5557e7695b80 finish 2019-12-07T19:33:58.625+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.625+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:34c0e826:::osdmap.19:0# 0x0~0 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e20) pg 0x5557e6e89000 2019-12-07T19:33:58.625+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:33:58.625+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:58.625+0000 7f2b784e2700 10 osd.6 19 dequeue_op 0x5557e7695e40 prio 63 cost 0 latency 0.000178 osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 pg pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_request waiting_for_map client.4540 not empty, queueing 2019-12-07T19:33:58.625+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e74d9ab0 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e74d97a0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.625+0000 7f2b784e2700 10 osd.6 19 dequeue_op 0x5557e7695e40 finish 2019-12-07T19:33:58.625+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.625+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e74d9810 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x5557e74d8c40 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:33:58.625+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:34c0e826:::osdmap.19:0# 0x0~140c = 5132 2019-12-07T19:33:58.625+0000 7f2b88017700 10 osd.6 19 add_map_bl 19 5132 bytes 2019-12-07T19:33:58.625+0000 7f2b88017700 20 osd.6 19 got_full_map 20, nothing requested 2019-12-07T19:33:58.625+0000 7f2b88017700 10 osd.6 19 handle_osd_map got inc map for epoch 21 2019-12-07T19:33:58.625+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3180e826:::osdmap.20:0# 0x0~0 2019-12-07T19:33:58.625+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3180e826:::osdmap.20:0# key 0x7f7fffffffffffffff3180e8'&!osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.625+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:58.625+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3180e826:::osdmap.20:0# 0x0~0 = -2 2019-12-07T19:33:58.625+0000 7f2b88017700 20 osd.6 19 got_full_map 21, nothing requested 2019-12-07T19:33:58.625+0000 7f2b88017700 20 osd.6 19 handle_osd_map pg_num_history pg_num_history(e21 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:58.625+0000 7f2b88017700 10 snap_mapper.record_purged_snaps purged_snaps {20={},21={}} 2019-12-07T19:33:58.625+0000 7f2b88017700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:58.625+0000 7f2b88017700 10 osd.6 19 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e21 [1,21] lci=[13,21]) 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e9b10000 seq 12 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:cd1b972f:::inc_osdmap.20:0# key 0x7f7fffffffffffffffcd1b97'/!inc_osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:58.626+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1080 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24d 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10000 1 blobs 2019-12-07T19:33:58.626+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x31000~1000/1000,1000,0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x31000~1000] 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e6f54380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6930 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e6f54380 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e75c6930) get_ref 0x0~24d Blob(0x5557e75c6930 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e6f54380 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e75c6930) get_ref init 0x1000, 1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x5557e75c6930 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x5557e6f54380 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e6f54398 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x24d 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d = 0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3180e826:::osdmap.20:0# key 0x7f7fffffffffffffff3180e8'&!osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:58.626+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3180e826:::osdmap.20:0# 0x0~140c 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1081 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3180e826:::osdmap.20:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~40c 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10000 2 blobs 2019-12-07T19:33:58.626+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x32000~2000/1000,2000,0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x32000~2000] 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b56000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b56000 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56070 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b56000) get_ref 0x0~1000 Blob(0x5557e9b56000 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56070 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b56000) get_ref init 0x1000, 1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e9b56000 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9b56070 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e9b56088 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b560e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b560e0 blob([!~1000,0x33000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56150 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b560e0) get_ref 0x1000~40c Blob(0x5557e9b560e0 blob([!~1000,0x33000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56150 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b560e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x5557e9b560e0 blob([!~1000,0x33000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x5557e9b56150 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e9b56168 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x140c 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3180e826:::osdmap.20:0# 0x0~140c = 0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c21b972f:::inc_osdmap.21:0# key 0x7f7fffffffffffffffc21b97'/!inc_osdmap.21!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:58.626+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1082 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24d 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10000 1 blobs 2019-12-07T19:33:58.626+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x34000~1000/1000,1000,0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x34000~1000] 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b562a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b562a0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56310 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b562a0) get_ref 0x0~24d Blob(0x5557e9b562a0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56310 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b562a0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x5557e9b562a0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x5557e9b56310 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e9b56328 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x24d 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d = 0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3a80e826:::osdmap.21:0# key 0x7f7fffffffffffffff3a80e8'&!osdmap.21!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:33:58.626+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3a80e826:::osdmap.21:0# 0x0~140c 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1083 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3a80e826:::osdmap.21:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~40c 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10000 2 blobs 2019-12-07T19:33:58.626+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x35000~2000/1000,2000,0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x35000~2000] 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b563f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b563f0 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56460 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b563f0) get_ref 0x0~1000 Blob(0x5557e9b563f0 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56460 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b563f0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e9b563f0 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9b56460 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e9b56478 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b564d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b564d0 blob([!~1000,0x36000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56540 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b564d0) get_ref 0x1000~40c Blob(0x5557e9b564d0 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56540 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e9b564d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x5557e9b564d0 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x5557e9b56540 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e9b56558 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x140c 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3a80e826:::osdmap.21:0# 0x0~140c = 0 2019-12-07T19:33:58.626+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~26 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10000 0 blobs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:58.626+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:58.626+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:58.626+0000 7f2b88017700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10000 0 blobs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10000 cost 41203 (7 ios * 4000 + 13203 bytes) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10000 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7606f00,0x5557e7607180,0x5557e7607400,0x5557e7607680 shared_blobs 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607008) update #-1:cd1b972f:::inc_osdmap.20:0# 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:cd1b972f:::inc_osdmap.20:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607288) update #-1:3180e826:::osdmap.20:0# 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607288) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3180e826:::osdmap.20:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607508) update #-1:c21b972f:::inc_osdmap.21:0# 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c21b972f:::inc_osdmap.21:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607788) update #-1:3a80e826:::osdmap.21:0# 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore.extentmap(0x5557e7607788) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3a80e826:::osdmap.21:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:58.626+0000 7f2b88017700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10000 allocated 0x[31000~6000] released 0x[] 2019-12-07T19:33:58.626+0000 7f2b88017700 10 freelist allocate 0x31000~6000 2019-12-07T19:33:58.626+0000 7f2b88017700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 prepare 2019-12-07T19:33:58.626+0000 7f2b88017700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b10000 2019-12-07T19:33:58.626+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 aio_wait 2019-12-07T19:33:58.626+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10000 2019-12-07T19:33:58.626+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 io_done 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1915 < min_flush_size 524288 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9026 < min_flush_size 524288 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606f00 had 1 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7607180 had 1 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7607400 had 1 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7607680 had 1 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860725760 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998762) bluestore 86 GiB free (0.959997), bluefs_ratio 0.039953 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9085 < min_flush_size 524288 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 9085 < min_flush_size 524288 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x1f885~237d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x1f885 0x1f885~237d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:56.615551+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _allocate len 0x1c02 from 0 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _allocate len 0x1c02 from 1 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,acd100000 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd130000~10000/10000,10000,acd100000 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x30000 to include allocated 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 24 (was clean) 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xf885 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x885 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xc02 and padding block with 0x3fe 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x30000 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x21c02 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (24) on file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]), flushing log 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 24 len 0x3c crc 0xc145b3d8) 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x17000~1000 to file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x17000 0x17000~1000 to file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x17000 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x18000 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:58.626+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000021 2019-12-07T19:33:58.626+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.627+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 24 2019-12-07T19:33:58.627+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.627+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 1 in 0.000509283s (0.000046049s flush + 0.000463234s kv commit) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10000> 2019-12-07T19:33:58.627+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <0x5557e79ede00> 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 kv_submitted 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10000 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e9b10000 osr 0x5557e76ce500 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e9b5adc0) prepare_write seq 10 0x1b000~1000 crc 1f5a496 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e9b5adc0) prepare_write seq 10 0x2000~1000 crc 8e45f3a8 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 deferred_cleanup 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6 19 _committed_osd_maps 20..21 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121800 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7556500,0x5557e7556780,0x5557e7556a00,0x5557e7556c80 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c61d8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aaf30 space 0x5557e75c61d8 0x0~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6 19 advance to epoch 20 (<= last 21 <= newest_map 21) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c6328 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab050 space 0x5557e75c6328 0x0~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c6408 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa120 space 0x5557e75c6408 0x1000~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c65c8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aaa20 space 0x5557e75c65c8 0x0~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c6718 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aaab0 space 0x5557e75c6718 0x0~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c67f8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aab40 space 0x5557e75c67f8 0x1000~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121800 done 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 deferred_cleanup 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121800 [] 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6 20 advance to epoch 21 (<= last 21 <= newest_map 21) 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 deferred_cleanup 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121500 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7557180,0x5557e7557400 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d8638 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa750 space 0x5557e74d8638 0x0~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d9f98 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aacf0 space 0x5557e74d9f98 0x0~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 7 osd.6 21 consume_map version 21 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a29908 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab0e0 space 0x5557e7a29908 0x1000~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.8s5 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 done 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.1s2 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92600 deferred_cleanup 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121500 [] 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.9s5 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.2s3 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.as1 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 deferred_cleanup 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.bs1 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92600 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7606000,0x5557e7606780 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d8c58 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa360 space 0x5557e74d8c58 0x1000~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d97b8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab950 space 0x5557e74d97b8 0x0~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a28a28 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa990 space 0x5557e7a28a28 0x0~1000 writing nocache) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.3s5 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92600 done 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.cs0 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10000 deferred_queued 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92600 [] 2019-12-07T19:33:58.627+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.5s1 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.6s1 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.es5 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.7s1 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 identify_splits_and_merges 1.fs5 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 prune_pg_created 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 21 prune_pg_created pruning 1.c 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6:0.consume_map 21 (was 19) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:0.consume_map consume_map 1.8s5 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6:1.consume_map 21 (was 19) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.1s2 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.9s5 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6:2.consume_map 21 (was 19) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.2s3 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.as1 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6:3.consume_map 21 (was 19) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.bs1 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.3s5 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6:4.consume_map 21 (was 19) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:4.consume_map consume_map 1.cs0 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6:5.consume_map 21 (was 19) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:5.consume_map consume_map 1.5s1 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6:6.consume_map 21 (was 19) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.6s1 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.es5 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6:7.consume_map 21 (was 19) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.7s1 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.fs5 2019-12-07T19:33:58.627+0000 7f2b85cfd700 10 osd.6 21 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.8s5 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.1s2 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.9s5 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e769f000 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.2s3 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6:0.update_pg_epoch 1.8s5 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.as1 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 19 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,4,5,2,3,6]/[1,4,5,2,3,6] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.bs1 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 20 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.3s5 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 20 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e767f000 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.8s5_head options 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e7957000 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 20 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e767c000 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 20 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 20 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.cs0 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.5s1 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 20 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6:0.update_pg_epoch 1.8s5 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6:2.update_pg_epoch 1.2s3 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6:1.update_pg_epoch 1.1s2 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 20 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,4,5,2,3,6]/[1,4,5,2,3,6] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6:1.update_pg_epoch 1.9s5 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.6s1 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 19 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 19 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.8s5_head options 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.es5 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 20 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.7s1 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 20 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 20 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.1s2_head options 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 20 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b85cfd700 15 osd.6 21 enqueue_peering_evt 1.fs5 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.2s3_head options 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 20 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b85cfd700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.9s5_head options 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 pg_epoch: 20 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 20 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 20 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e7954000 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 pg_epoch: 20 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 20 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6:3.update_pg_epoch 1.bs1 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 pg_epoch: 20 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 19 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6:3.update_pg_epoch 1.3s5 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6:2.update_pg_epoch 1.2s3 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 20 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 19 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 20 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.bs1_head options 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 20 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e769c000 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6:7.update_pg_epoch 1.7s1 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 20 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 20 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.3s5_head options 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 19 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6:4.update_pg_epoch 1.cs0 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 pg_epoch: 20 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 20 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 20 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 pg_epoch: 20 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6:6.update_pg_epoch 1.6s1 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.2s3_head options 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 pg_epoch: 20 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 19 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map [6,5,4,1,2,3]/[6,5,4,1,2,3] -- 6/6 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 20 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:33:58.627+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.7s1_head options 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 pg_epoch: 20 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 20 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 pg_epoch: 20 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 20 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active advmap 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 pg_epoch: 20 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 20 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 20 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6:3.update_pg_epoch 1.bs1 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6:3.update_pg_epoch 1.3s5 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.6s1_head options 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 20 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 20 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 20 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 20 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6:7.update_pg_epoch 1.7s1 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 20 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.cs0_head options 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 20 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 20 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.bs1_head options 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 20 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 20 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 pg_epoch: 20 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.3s5_head options 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.7s1_head options 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6:6.update_pg_epoch 1.6s1 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b85cfd700 7 osd.6 21 activate_map version 21 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 pg_epoch: 20 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 pg_epoch: 20 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] agent_stop 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e766d000 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e764f000 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 20 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6:1.update_pg_epoch 1.1s2 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 20 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6:2.update_pg_epoch 1.as1 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 20 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 20 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6:1.update_pg_epoch 1.9s5 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6:4.update_pg_epoch 1.cs0 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 19 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 20 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6:7.update_pg_epoch 1.fs5 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 20 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map [6,5,4,1,2,3]/[6,5,4,1,2,3] -- 6/6 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 19 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.1s2_head options 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active advmap 2019-12-07T19:33:58.627+0000 7f2b7ace7700 20 osd.6 21 agent_entry empty queue 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 20 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.9s5_head options 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.as1_head options 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b74cdb700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 20 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.fs5_head options 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.cs0_head options 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 20 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 20 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 pg_epoch: 20 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 20 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 pg_epoch: 20 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] agent_stop 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 20 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6:7.update_pg_epoch 1.fs5 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6:2.update_pg_epoch 1.as1 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 20 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 20 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] activate_map 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.fs5_head options 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.as1_head options 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.6s1_head options 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] Active: kicking snap trim 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b744da700 7 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: handle ActMap primary 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:12 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] requeue_map_waiters client.4540 0x5557e76951e0,0x5557e76954a0,0x5557e7695760,0x5557e76958c0,0x5557e7695b80,0x5557e7695e40 2019-12-07T19:33:58.627+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e766a000 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6:6.update_pg_epoch 1.es5 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e21) 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 19 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) 2019-12-07T19:33:58.627+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.es5_head options 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 20 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e21) 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 20 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6:6.update_pg_epoch 1.es5 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e21) queued 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 20 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.627+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.es5_head options 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e76951e0 prio 63 cost 32768 latency 0.002322 osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) queued 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 21 send_incremental_map 20 -> 21 to 0x5557e6f87800 172.21.15.131:0/224485236 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~0 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:33:58.627+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b562a0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x5557e9b56310 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d = 589 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 21 add_map_inc_bl 21 589 bytes 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x5557e764c000 2019-12-07T19:33:58.627+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_map(21..21 src has 1..21) v4 -- 0x5557e760c6c0 con 0x5557e6f87800 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6:5.update_pg_epoch 1.5s1 19 -> 20 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 21 maybe_share_map con 0x5557e6f87800 172.21.15.131:0/224485236 map epoch 20 -> 21 (shared) 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 19 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [2,6,3,4,1,5]/[2,6,3,4,1,5] -- 2/2 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 20 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 20 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.5s1_head options 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 20 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 pg_epoch: 20 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 pg_epoch: 20 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:1 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 20 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6:5.update_pg_epoch 1.5s1 20 -> 21 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 20 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [2,6,3,4,1,5]/[2,6,3,4,1,5] -- 2/2 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.5s1_head options 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7f2b73cd9700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat] snapc 0=[] ondisk+write+known_if_redirected e20) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: obc NOT found in cache: 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.627+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# _ 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3c88320e:::smithi13140796-1:head# key 0x8080000000000000013c88320e21736d'ithi13140796-1!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# _ = -2 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: no obc for soid 1:3c88320e:::smithi13140796-1:head but can_create 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] create_object_context 0x5557e760c900 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] populate_obc_watchers 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e760c900 1:3c88320e:::smithi13140796-1:head rwstate(none n=0 w=0) oi: 1:3c88320e:::smithi13140796-1:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x5557e7967680 snapset: 0=[]:{} 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.627+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op oi 1:3c88320e:::smithi13140796-1:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op obc obc(1:3c88320e:::smithi13140796-1:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx 0x5557e9b14300 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat] ov 0'0 av 21'1 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat] 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op append 0~32768 in=32768b 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op 1:3c88320e:::smithi13140796-1:head [write 0~32768 in=32768b] 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op write 0~32768 in=32768b 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] clean_regions modifiedclean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op stat 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] stat oi has 32768 0.000000 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] make_writeable 1:3c88320e:::smithi13140796-1:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] setting DIRTY flag 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] make_writeable 1:3c88320e:::smithi13140796-1:head done, snapset=0=[]:{} 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_ctx 1:3c88320e:::smithi13140796-1:head 0x5557e9b14300 op modify 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.625551+0000 2019-12-07T19:33:58.627+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.627+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_ctx object 1:3c88320e:::smithi13140796-1:head marks clean_regions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x5557e76ca500 result 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] op order client.4540 tid 1 (first) 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] new_repop rep_tid 0 on osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] new_repop: repgather(0x5557e7967320 0'0 rep_tid=0 committed?=0 r=0) 2019-12-07T19:33:58.628+0000 7f2b784e2700 7 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] issue_repop rep_tid 0 o 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:3c88320e:::smithi13140796-1:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:1 client_op=osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3c88320e:::smithi13140796-1:head# key 0x8080000000000000013c88320e21736d'ithi13140796-1!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): 1:3c88320e:::smithi13140796-1:head projected size 32768 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] start_rmw: Op(1:3c88320e:::smithi13140796-1:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:1 client_op=osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[0~32768]}) 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:3c88320e:::smithi13140796-1:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:1 client_op=osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[0~32768]}) 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:3c88320e:::smithi13140796-1:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:1 client_op=osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[0~32768]}) 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(0~32768:1) ) 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] generate_transactions: 1:3c88320e:::smithi13140796-1:head, current size is 0 buffers are {0~32768(32768)} 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): adding buffer_update 0,32768 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): to_append: {0~32768(32768)} 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): appending 0~32768 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] encode_and_write: 1:3c88320e:::smithi13140796-1:head new_size 32768 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): 1:3c88320e:::smithi13140796-1:head resetting hinfo to logical size 32768 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): marking append 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(0~32768:1) ) 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:3c88320e:::smithi13140796-1:head={0~32768(32768)}} 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:3c88320e:::smithi13140796-1:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:1 client_op=osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[0~32768]}) 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:3c88320e:::smithi13140796-1:head=[0~32768]} 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:3c88320e:::smithi13140796-1:head,{0~32768(32768)} 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 send_incremental_map 19 -> 21 to 0x5557e79cac00 v2:172.21.15.131:6823/37078 2019-12-07T19:33:58.628+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~0 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c6930 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x5557e6f54380 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d = 589 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 add_map_inc_bl 20 589 bytes 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(20..21 src has 1..21) v4 -- 0x5557e74cafc0 con 0x5557e79cac00 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 maybe_share_map con 0x5557e79cac00 v2:172.21.15.131:6823/37078 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:1, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e763c800 con 0x5557e79cac00 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 send_incremental_map 19 -> 21 to 0x5557e7519800 v2:172.21.15.131:6804/37077 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(20..21 src has 1..21) v4 -- 0x5557e9b206c0 con 0x5557e7519800 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 maybe_share_map con 0x5557e7519800 v2:172.21.15.131:6804/37077 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:1, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e763f000 con 0x5557e7519800 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 send_incremental_map 19 -> 21 to 0x5557e79ca800 v2:172.21.15.131:6812/37076 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(20..21 src has 1..21) v4 -- 0x5557e9b20900 con 0x5557e79ca800 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 maybe_share_map con 0x5557e79ca800 v2:172.21.15.131:6812/37076 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:1, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e763f800 con 0x5557e79ca800 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 send_incremental_map 19 -> 21 to 0x5557e79cbc00 v2:172.21.15.109:6810/36938 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(20..21 src has 1..21) v4 -- 0x5557e7adf680 con 0x5557e79cbc00 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 maybe_share_map con 0x5557e79cbc00 v2:172.21.15.109:6810/36938 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=0, reqid=client.4540.0:1, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e7a0e000 con 0x5557e79cbc00 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 send_incremental_map 19 -> 21 to 0x5557e79ca000 v2:172.21.15.109:6802/36936 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(20..21 src has 1..21) v4 -- 0x5557e7adf8c0 con 0x5557e79ca000 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 21 maybe_share_map con 0x5557e79ca000 v2:172.21.15.109:6802/36936 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.628+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:1, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e79e2000 con 0x5557e79ca000 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 earliest_dup_version = 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 trim 21'1 (0'0) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:1 2019-12-07T19:33:58.625551+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:1 2019-12-07T19:33:58.625551+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'1 (0'0) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:1 2019-12-07T19:33:58.625551+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 1 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.628+0000 7f2b784e2700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7692f00 seq 6 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3c88320e:::smithi13140796-1:head# key 0x8080000000000000013c88320e21736d'ithi13140796-1!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.628+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1084 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# = 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2 keys 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2 keys = 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.628+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x0~2000 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 0#1:3c88320e:::smithi13140796-1:head# 0x0~2000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.628+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~2000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692f00 1 blobs 2019-12-07T19:33:58.628+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x37000~2000/1000,2000,0 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x37000~2000] 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b578f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b578f0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57880 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b578f0) get_ref 0x0~2000 Blob(0x5557e9b578f0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57880 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b578f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9b578f0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57880 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b57898 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x2000 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e76cab08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x0~2000 = 0 2019-12-07T19:33:58.629+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.629+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692f00 cost 22619 (3 ios * 4000 + 10619 bytes) 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692f00 onodes 0x5557e76caa00 shared_blobs 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e76cab08) update 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e76cab08) update inline shard 22 bytes from 1 extents 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:3c88320e:::smithi13140796-1:head# is 439 (411 bytes onode + 2 bytes spanning blobs + 26 bytes inline extents) 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692f00 allocated 0x[37000~2000] released 0x[] 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 freelist allocate 0x37000~2000 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 prepare 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692f00 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'1 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x5557e7967320 21'1 rep_tid=0 committed?=0 r=0) 2019-12-07T19:33:58.629+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e76951e0 finish 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e76954a0 prio 63 cost 62 latency 0.003840 osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:2 1.cs0 1.704c113c (undecoded) ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.629+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 aio_wait 2019-12-07T19:33:58.629+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692f00 2019-12-07T19:33:58.629+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 io_done 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x5557e760c900 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e760c900 1:3c88320e:::smithi13140796-1:head rwstate(excl n=1 w=0) oi: 1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967680 snapset: 0=[]:{} 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.629+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e76954a0 finish 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e21) queued 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1963 < min_flush_size 524288 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695760 prio 63 cost 0 latency 0.003919 osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 pg pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:3 1.cs0 1.704c113c (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76caa00 had 1 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1986 < min_flush_size 524288 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1986 < min_flush_size 524288 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x21c02~7c2 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x5557e760c900 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x21c02 0x21c02~7c2 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x1c02 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xc02 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e760c900 1:3c88320e:::smithi13140796-1:head rwstate(excl n=1 w=1) oi: 1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967680 snapset: 0=[]:{} 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x3c4 and padding block with 0xc3c 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.629+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695760 finish 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) queued 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 32768 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e76958c0 prio 63 cost 32768 latency 0.003930 osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:4 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x223c4 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.629+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat] snapc 0=[] ondisk+write+known_if_redirected e20) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: obc NOT found in cache: 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.629+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# _ 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000169200s (0.000044908s flush + 0.000124292s kv commit) 2019-12-07T19:33:58.629+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:33f5bf01:::smithi13140796-2:head# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.629+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.629+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# _ = -2 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: no obc for soid 1:33f5bf01:::smithi13140796-2:head but can_create 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] create_object_context 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] populate_obc_watchers 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692f00> 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.629+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 kv_submitted 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692f00 2019-12-07T19:33:58.629+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 finishing 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692f00 onodes 0x5557e76caa00 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b57898 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e79aaea0 space 0x5557e9b57898 0x0~2000 writing nocache) 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692f00 done 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.629+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692f00 [] 2019-12-07T19:33:58.629+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.629+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.629+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op obc obc(1:33f5bf01:::smithi13140796-2:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 0x5557e70b7800 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat] ov 0'0 av 21'2 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat] 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op append 0~32768 in=32768b 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [write 0~32768 in=32768b] 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op write 0~32768 in=32768b 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] clean_regions modifiedclean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op stat 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] stat oi has 32768 0.000000 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] setting DIRTY flag 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=0=[]:{} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x5557e70b7800 op modify 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.625910+0000 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x5557e76cac80 result 0 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op order client.4540 tid 4 (first) 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop rep_tid 1 on osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop: repgather(0x5557e628b200 0'0 rep_tid=1 committed?=0 r=0) 2019-12-07T19:33:58.629+0000 7f2b744da700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] issue_repop rep_tid 1 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:4 client_op=osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.629+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.629+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:33f5bf01:::smithi13140796-2:head# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 32768 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:4 client_op=osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[0~32768]}) 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:4 client_op=osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[0~32768]}) 2019-12-07T19:33:58.629+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:4 client_op=osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[0~32768]}) 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~32768:2) Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(0~32768:1) ) 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 0 buffers are {0~32768(32768)} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): adding buffer_update 0,32768 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_append: {0~32768(32768)} 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): appending 0~32768 2019-12-07T19:33:58.629+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] encode_and_write: 1:33f5bf01:::smithi13140796-2:head new_size 32768 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 32768 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): marking append 0 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~32768:2) Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(0~32768:1) ) 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:head={0~32768(32768)}} 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:4 client_op=osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[0~32768]}) 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:head=[0~32768]} 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{0~32768(32768)} 2019-12-07T19:33:58.630+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:4, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e79e3000 con 0x5557e79cac00 2019-12-07T19:33:58.630+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:4, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e79e2800 con 0x5557e7519800 2019-12-07T19:33:58.630+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:4, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e79e4800 con 0x5557e79ca800 2019-12-07T19:33:58.630+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:4, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e79e5000 con 0x5557e79cbc00 2019-12-07T19:33:58.630+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:4, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e79e5800 con 0x5557e79ca000 2019-12-07T19:33:58.630+0000 7f2b744da700 20 earliest_dup_version = 0 2019-12-07T19:33:58.630+0000 7f2b744da700 20 trim 21'2 (0'0) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:4 2019-12-07T19:33:58.625910+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log log((0'0,21'1], crt=0'0) [21'2 (0'0) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:4 2019-12-07T19:33:58.625910+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'2 (0'0) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:4 2019-12-07T19:33:58.625910+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 2 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.630+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.630+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7a92f00 seq 7 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:33f5bf01:::smithi13140796-2:head# key 0x80800000000000000133f5bf0121736d'ithi13140796-2!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.630+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1085 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# = 0 2019-12-07T19:33:58.630+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:33:58.630+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.630+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~2000 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 0#1:33f5bf01:::smithi13140796-2:head# 0x0~2000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~2000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92f00 1 blobs 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.630+0000 7f2b744da700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:33:58.630+0000 7f2b744da700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x39000~2000/1000,2000,0 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x39000~2000] 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7620700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7620690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7620700 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620690 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore.blob(0x5557e7620700) get_ref 0x0~2000 Blob(0x5557e7620700 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620690 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore.blob(0x5557e7620700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7620700 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620690 sbid 0x0)) 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e76206a8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x2000 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore.extentmap(0x5557e76cb508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~2000 = 0 2019-12-07T19:33:58.630+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.630+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92f00 cost 21811 (3 ios * 4000 + 9811 bytes) 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92f00 onodes 0x5557e76cb400 shared_blobs 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore.extentmap(0x5557e76cb508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore.extentmap(0x5557e76cb508) update inline shard 22 bytes from 1 extents 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 439 (411 bytes onode + 2 bytes spanning blobs + 26 bytes inline extents) 2019-12-07T19:33:58.630+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92f00 allocated 0x[39000~2000] released 0x[] 2019-12-07T19:33:58.630+0000 7f2b744da700 10 freelist allocate 0x39000~2000 2019-12-07T19:33:58.630+0000 7f2b744da700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 prepare 2019-12-07T19:33:58.630+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a92f00 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'2 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x5557e628b200 21'2 rep_tid=1 committed?=0 r=0) 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e76958c0 finish 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695b80 prio 63 cost 62 latency 0.005070 osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:5 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(excl n=1 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.630+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.630+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695b80 finish 2019-12-07T19:33:58.630+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.630+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 27 ==== MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 14256+0+0 (unknown 0 0 0) 0x5557e70a7000 con 0x5557e79cac00 2019-12-07T19:33:58.630+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695600 prio 127 cost 0 latency 0.000042 epoch 21 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.630+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.630+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.630+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.630+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.630+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e767c000 2019-12-07T19:33:58.630+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 28 ==== MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 22448+0+0 (unknown 0 0 0) 0x5557e7a10000 con 0x5557e79cac00 2019-12-07T19:33:58.630+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.630+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.630+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.630+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e7695600 prio 127 cost 0 latency 0.000093 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.630+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.630+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.630+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695e40 prio 63 cost 0 latency 0.005272 osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 pg pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.630+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:6 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.630+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.630+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.630+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.630+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.630+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 29 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:19, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 30640+0+0 (unknown 0 0 0) 0x5557e9ba0000 con 0x5557e79cac00 2019-12-07T19:33:58.630+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.630+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:19, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.630+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:19, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.630+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.630+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.630+0000 7f2b75cdd700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.630+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.630+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.630+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(excl n=1 w=1) oi: 1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.630+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.630+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.630+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.630+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695e40 finish 2019-12-07T19:33:58.630+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 18 ==== MOSDECSubOpWrite(1.6s1 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 18352+0+0 (crc 0 0 0) 0x5557e9ba0800 con 0x5557e79ca800 2019-12-07T19:33:58.630+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.630+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c42c0 prio 127 cost 0 latency 0.000022 epoch 20 MOSDECSubOpWrite(1.6s1 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.630+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:58.630+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e20) 2019-12-07T19:33:58.630+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7631b00 seq 5 2019-12-07T19:33:58.630+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.9s5_head 0x5557e75e2b40) get_onode oid 5#1:9694a0e5:::smithi13140796-3:head# key 0x8580000000000000019694a0e521736d'ithi13140796-3!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.630+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1086 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# = 0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.631+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e766a000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~3000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7631b00 2 blobs 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000225 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:13 2019-12-07T19:33:58.627318+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 fbmap_alloc 0x5557e6282e00 allocate 0x3000/1000,3000,0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x3b000~3000/1000,3000,0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x3b000~3000] 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668d90 blob([0x3b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7e30 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:13 2019-12-07T19:33:58.627318+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7668d90) get_ref 0x0~2000 Blob(0x5557e7668d90 blob([0x3b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7e30 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7668d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7668d90 blob([0x3b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7e30 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e75c7e48 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c69a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7668e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.631+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c69a0 blob([0x3d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668e00 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.blob(0x5557e75c69a0) get_ref 0x0~1000 Blob(0x5557e75c69a0 blob([0x3d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668e00 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.blob(0x5557e75c69a0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~1000: 0x0~1000 Blob(0x5557e75c69a0 blob([0x3d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7668e00 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e7668e18 in 0x5557e6269a40) _discard 0x0~1000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x3000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e76cba08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 = 0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e9b11500 seq 5 2019-12-07T19:33:58.631+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.es5_head 0x5557e75e34a0) get_onode oid 5#1:733d7474:::smithi13140796-5:head# key 0x858000000000000001733d74't!smithi13140796-5!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.es5_head 5#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1087 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.es5_head 5#1:733d7474:::smithi13140796-5:head# = 0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.es5_head 5#1:733d7474:::smithi13140796-5:head# 2 keys 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.es5_head 5#1:733d7474:::smithi13140796-5:head# 2 keys = 0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.es5_head 5#1:733d7474:::smithi13140796-5:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631b00 cost 26715 (3 ios * 4000 + 14715 bytes) 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631b00 onodes 0x5557e76cb900 shared_blobs 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.es5_head 5#1:733d7474:::smithi13140796-5:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e76cba08) update 5#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.es5_head 5#1:733d7474:::smithi13140796-5:head# 0x0~5000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e76cba08) update inline shard 38 bytes from 2 extents 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:733d7474:::smithi13140796-5:head# 0x0~5000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:9694a0e5:::smithi13140796-3:head# is 455 (411 bytes onode + 2 bytes spanning blobs + 42 bytes inline extents) 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~5000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631b00 allocated 0x[3b000~3000] released 0x[] 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 freelist allocate 0x3b000~3000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b11500 3 blobs 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 prepare 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7631b00 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.631+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 26 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (unknown 0 0 0) 0x5557e760c6c0 con 0x5557e79cac00 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.631+0000 7f2b834f8700 20 osd.6 21 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.631+0000 7f2b834f8700 10 osd.6 21 do_waiters -- start 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x5000/1000,5000,0 2019-12-07T19:33:58.631+0000 7f2b834f8700 10 osd.6 21 do_waiters -- finish 2019-12-07T19:33:58.631+0000 7f2b834f8700 20 osd.6 21 _dispatch 0x5557e760c6c0 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x3e000~5000/1000,5000,0 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x3e000~5000] 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76697a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e76203f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.631+0000 7f2b834f8700 3 osd.6 21 handle_osd_map epochs [20,21], i have 21, src has [1,21] 2019-12-07T19:33:58.631+0000 7f2b834f8700 10 osd.6 21 no new maps here, dropping 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76697a0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76203f0 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e76697a0) get_ref 0x0~2000 Blob(0x5557e76697a0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76203f0 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e76697a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.631+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 17 ==== osd_map(20..20 src has 1..20) v4 ==== 633+0+0 (crc 0 0 0) 0x5557e7ade000 con 0x5557e79ca800 2019-12-07T19:33:58.631+0000 7f2b834f8700 20 osd.6 21 OSD::ms_dispatch: osd_map(20..20 src has 1..20) v4 2019-12-07T19:33:58.631+0000 7f2b834f8700 10 osd.6 21 do_waiters -- start 2019-12-07T19:33:58.631+0000 7f2b834f8700 10 osd.6 21 do_waiters -- finish 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e76697a0 blob([0x3e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76203f0 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b834f8700 20 osd.6 21 _dispatch 0x5557e7ade000 osd_map(20..20 src has 1..20) v4 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7620408 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.631+0000 7f2b834f8700 3 osd.6 21 handle_osd_map epochs [20,20], i have 21, src has [1,20] 2019-12-07T19:33:58.631+0000 7f2b834f8700 10 osd.6 21 no new maps here, dropping 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7669a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668d20 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669a40 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e7668d20) get_ref 0x0~2000 Blob(0x5557e7668d20 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669a40 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e7668d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7668d20 blob([0x40000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7669a40 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7669a58 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669ab0 blob([0x42000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28000 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e7669ab0) get_ref 0x0~1000 Blob(0x5557e7669ab0 blob([0x42000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28000 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e7669ab0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~1000: 0x0~1000 Blob(0x5557e7669ab0 blob([0x42000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7a28000 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e7695600 finish 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7a28018 in 0x5557e6269ea0) _discard 0x0~1000 2019-12-07T19:33:58.631+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x5000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.631+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 aio_wait 2019-12-07T19:33:58.631+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92f00 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.es5_head 5#1:733d7474:::smithi13140796-5:head# 0x0~5000 = 0 2019-12-07T19:33:58.631+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 io_done 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.es5_head 5#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.es5_head 5#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:58.631+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 aio_wait 2019-12-07T19:33:58.631+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631b00 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.631+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 io_done 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11500 cost 38907 (4 ios * 4000 + 22907 bytes) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11500 onodes 0x5557e9bc0280 shared_blobs 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0388) update 5#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0388) update inline shard 57 bytes from 3 extents 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:733d7474:::smithi13140796-5:head# is 475 (412 bytes onode + 2 bytes spanning blobs + 61 bytes inline extents) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11500 allocated 0x[3e000~5000] released 0x[] 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 freelist allocate 0x3e000~5000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 prepare 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b11500 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4000 finish 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e20) queued 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e20) pg 0x5557e769c000 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c42c0 prio 127 cost 0 latency 0.000592 MOSDECSubOpWrite(1.6s1 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 20/18 ECSubWrite(tid=0, reqid=client.4540.0:10, at_version=20'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [20'1 (0'0) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:10 2019-12-07T19:33:58.626763+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 20'1 (0'0) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:10 2019-12-07T19:33:58.626763+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.631+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 20'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb400 had 1 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3128 < min_flush_size 524288 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9b11b00 seq 5 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb900 had 1 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3151 < min_flush_size 524288 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3151 < min_flush_size 524288 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) get_onode oid 1#1:657ac0fe:::smithi13140796-4:head# key 0x818000000000000001657ac0fe21736d'ithi13140796-4!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1088 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x223c4~c4f to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# = 0 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x223c4 0x223c4~c4f to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 2 keys 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 2 keys = 0 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x23c4 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x3c4 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x13 and padding block with 0xfed 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 0x0~4000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:657ac0fe:::smithi13140796-4:head# 0x0~4000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~4000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b11b00 2 blobs 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x4000/1000,4000,0 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x43000~4000/1000,4000,0 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x43000~4000] 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a28a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a28a80 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621030 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e7a28a80) get_ref 0x0~2000 Blob(0x5557e7a28a80 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621030 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e7a28a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7a28a80 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621030 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7621048 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7620460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7620460 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28460 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e7620460) get_ref 0x0~2000 Blob(0x5557e7620460 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28460 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.blob(0x5557e7620460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7620460 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7a28460 sbid 0x0)) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7a28478 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x4000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0608) dirty_range mark inline shard dirty 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 0x0~4000 = 0 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x23013 2019-12-07T19:33:58.631+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.631+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000020.00000000000000000001' <- 0000000020.00000000000000000001 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000260250s (0.000050550s flush + 0.000209700s kv commit) 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11b00 cost 30811 (3 ios * 4000 + 18811 bytes) 2019-12-07T19:33:58.631+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11b00 onodes 0x5557e9bc0500 shared_blobs 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0608) update 1#1:657ac0fe:::smithi13140796-4:head# 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0608) update inline shard 41 bytes from 2 extents 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:657ac0fe:::smithi13140796-4:head# is 459 (412 bytes onode + 2 bytes spanning blobs + 45 bytes inline extents) 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11b00 allocated 0x[43000~4000] released 0x[] 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 freelist allocate 0x43000~4000 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 prepare 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b11b00 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92f00,0x5557e7631b00> 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.631+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 kv_submitted 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92f00 2019-12-07T19:33:58.631+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 finishing 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92f00 onodes 0x5557e76cb400 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e76206a8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e79ab680 space 0x5557e76206a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92f00 done 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.631+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92f00 [] 2019-12-07T19:33:58.631+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.631+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 kv_submitted 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631b00 2019-12-07T19:33:58.631+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 finishing 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631b00 onodes 0x5557e76cb900 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c7e48 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b585a0 space 0x5557e75c7e48 0x0~2000 writing nocache) 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7668e18 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b58630 space 0x5557e7668e18 0x0~1000 writing nocache) 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631b00 done 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 20'1 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:58.631+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631b00 [] 2019-12-07T19:33:58.631+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.631+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.631+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c42c0 finish 2019-12-07T19:33:58.631+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.632+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:19, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.632+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:19, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.632+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 127 cost 0 latency 0.001357 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:19, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.632+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:19, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.632+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:19 2019-12-07T19:33:58.628809+0000 0 ObjectCleanRegions clean_offsets: [114688~18446744073709436927], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.632+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:19 2019-12-07T19:33:58.628809+0000 0 ObjectCleanRegions clean_offsets: [114688~18446744073709436927], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.632+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.632+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.632+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.632+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 7 ==== osd_op(client.4540.0:25 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 263+0+147456 (crc 0 0 0) 0x5557e9b54b00 con 0x5557e6f87800 2019-12-07T19:33:58.632+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.632+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4420 prio 63 cost 147456 latency 0.000020 epoch 21 osd_op(client.4540.0:25 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.632+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:25 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 147456 e21) 2019-12-07T19:33:58.632+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 8 ==== osd_op(client.4540.0:26 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 225+0+62 (crc 0 0 0) 0x5557e9b54dc0 con 0x5557e6f87800 2019-12-07T19:33:58.632+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c46e0 prio 63 cost 62 latency 0.000009 epoch 21 osd_op(client.4540.0:26 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.632+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:26 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 62 e21) 2019-12-07T19:33:58.632+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 9 ==== osd_op(client.4540.0:27 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 225+0+0 (crc 0 0 0) 0x5557e9b55080 con 0x5557e6f87800 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.632+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c49a0 prio 63 cost 0 latency 0.000007 epoch 21 osd_op(client.4540.0:27 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7a92c00 seq 5 2019-12-07T19:33:58.632+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:131421e8:::smithi13140796-7:head# key 0x858000000000000001131421e821736d'ithi13140796-7!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.632+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1089 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# = 0 2019-12-07T19:33:58.632+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:33:58.632+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.632+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0~7000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:131421e8:::smithi13140796-7:head# 0x0~7000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~7000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92c00 4 blobs 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.632+0000 7f2b764de700 10 fbmap_alloc 0x5557e6282e00 allocate 0x7000/1000,7000,0 2019-12-07T19:33:58.632+0000 7f2b764de700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x47000~7000/1000,7000,0 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x47000~7000] 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b57f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b57f80 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621180 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57f80) get_ref 0x0~2000 Blob(0x5557e9b57f80 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621180 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.632+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9b57f80 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621180 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e7621198 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b57420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b57420 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57ea0 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57420) get_ref 0x0~2000 Blob(0x5557e9b57420 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57ea0 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9b57420 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57ea0 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57eb8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b57e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b57e30 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57340 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57e30) get_ref 0x0~2000 Blob(0x5557e9b57e30 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57340 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9b57e30 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57340 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57358 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.632+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e765f200 con 0x5557e79cac00 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b577a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b577a0 blob([0x4d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57b90 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.blob(0x5557e9b577a0) get_ref 0x0~1000 Blob(0x5557e9b577a0 blob([0x4d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57b90 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.blob(0x5557e9b577a0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~1000: 0x0~1000 Blob(0x5557e9b577a0 blob([0x4d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9b57b90 sbid 0x0)) 2019-12-07T19:33:58.632+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57ba8 in 0x5557e6269960) _discard 0x0~1000 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:25 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 147456 e21) queued 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x7000 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.extentmap(0x5557e9bc1008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:25 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 147456 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0~7000 = 0 2019-12-07T19:33:58.632+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.632+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4420 prio 63 cost 147456 latency 0.000265 osd_op(client.4540.0:25 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:25 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._epoch' <- _epoch 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 51099 (5 ios * 4000 + 31099 bytes) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes 0x5557e9bc0f00 shared_blobs 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.extentmap(0x5557e9bc1008) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore.extentmap(0x5557e9bc1008) update inline shard 76 bytes from 4 extents 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 494 (412 bytes onode + 2 bytes spanning blobs + 80 bytes inline extents) 2019-12-07T19:33:58.632+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[47000~7000] released 0x[] 2019-12-07T19:33:58.632+0000 7f2b764de700 10 freelist allocate 0x47000~7000 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.632+0000 7f2b764de700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: obc NOT found in cache: 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.632+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a92c00 2019-12-07T19:33:58.632+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# _ 2019-12-07T19:33:58.632+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3f7a742d:::smithi13140796-9:head# key 0x8080000000000000013f7a74'-!smithi13140796-9!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.632+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.632+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# _ = -2 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: no obc for soid 1:3f7a742d:::smithi13140796-9:head but can_create 2019-12-07T19:33:58.632+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 30 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 42932+0+0 (unknown 0 0 0) 0x5557e79e3000 con 0x5557e79cac00 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] create_object_context 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.632+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4b00 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] populate_obc_watchers 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.632+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(none n=0 w=0) oi: 1:3f7a742d:::smithi13140796-9:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.632+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op obc obc(1:3f7a742d:::smithi13140796-9:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 0x5557e769aa00 2019-12-07T19:33:58.632+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat] ov 0'0 av 21'3 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.632+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.632+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat] 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op append 0~147456 in=147456b 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:3f7a742d:::smithi13140796-9:head [write 0~147456 in=147456b] 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op write 0~147456 in=147456b 2019-12-07T19:33:58.632+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 31 ==== MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 26544+0+0 (unknown 0 0 0) 0x5557e763c800 con 0x5557e79cac00 2019-12-07T19:33:58.632+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4c60 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.632+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] clean_regions modifiedclean_offsets: [147456~18446744073709404159], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op stat 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] stat oi has 147456 0.000000 2019-12-07T19:33:58.632+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 aio_wait 2019-12-07T19:33:58.632+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11500 2019-12-07T19:33:58.632+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 io_done 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:3f7a742d:::smithi13140796-9:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.632+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 aio_wait 2019-12-07T19:33:58.632+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11b00 2019-12-07T19:33:58.632+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 io_done 2019-12-07T19:33:58.632+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 aio_wait 2019-12-07T19:33:58.632+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] setting DIRTY flag 2019-12-07T19:33:58.632+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:3f7a742d:::smithi13140796-9:head done, snapset=0=[]:{} 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=3 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx 1:3f7a742d:::smithi13140796-9:head 0x5557e769aa00 op modify 2019-12-07T19:33:58.632+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.630729+0000 2019-12-07T19:33:58.632+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.632+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4b00 prio 127 cost 0 latency 0.000192 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx object 1:3f7a742d:::smithi13140796-9:head marks clean_regions clean_offsets: [147456~18446744073709404159], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x5557e76cb180 result 0 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.632+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1999 < min_flush_size 524288 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op order client.4540 tid 25 (first) 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop rep_tid 2 on osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0280 had 1 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop: repgather(0x5557e7967b00 0'0 rep_tid=2 committed?=0 r=0) 2019-12-07T19:33:58.632+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3982 < min_flush_size 524288 2019-12-07T19:33:58.632+0000 7f2b744da700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] issue_repop rep_tid 2 o 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.632+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0500 had 1 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.632+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6000 < min_flush_size 524288 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:3f7a742d:::smithi13140796-9:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:25 client_op=osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.632+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7665200 seq 5 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0f00 had 1 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.632+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6023 < min_flush_size 524288 2019-12-07T19:33:58.632+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.632+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6023 < min_flush_size 524288 2019-12-07T19:33:58.632+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.632+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.632+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3f7a742d:::smithi13140796-9:head# key 0x8080000000000000013f7a74'-!smithi13140796-9!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.632+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.7s1_head 0x5557e76a92c0) get_onode oid 1#1:ecc72bdb:::smithi13140796-10:head# key 0x818000000000000001ecc72bdb21736d'ithi13140796-10!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.632+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x23013~1787 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.632+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x23013 0x23013~1787 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.632+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.632+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1090 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# = 0 2019-12-07T19:33:58.632+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x3013 2019-12-07T19:33:58.632+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x13 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.632+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x79a and padding block with 0x866 2019-12-07T19:33:58.632+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:33:58.632+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:3f7a742d:::smithi13140796-9:head projected size 147456 2019-12-07T19:33:58.633+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~a000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.633+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] start_rmw: Op(1:3f7a742d:::smithi13140796-9:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:25 client_op=osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~147456]}) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7665200 5 blobs 2019-12-07T19:33:58.633+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:3f7a742d:::smithi13140796-9:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:25 client_op=osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~147456]}) 2019-12-07T19:33:58.633+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2479a 2019-12-07T19:33:58.633+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.633+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.633+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:3f7a742d:::smithi13140796-9:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:25 client_op=osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~147456]}) 2019-12-07T19:33:58.633+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.633+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~32768:2) Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(0~32768:1) Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 3 cleaned 0 in 0.000212819s (0.000044751s flush + 0.000168068s kv commit) 2019-12-07T19:33:58.633+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11500,0x5557e9b11b00,0x5557e7a92c00> 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 kv_submitted 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] generate_transactions: 1:3f7a742d:::smithi13140796-9:head, current size is 0 buffers are {0~147456(147456)} 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11500 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 finishing 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11500 onodes 0x5557e9bc0280 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7620408 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9b58870 space 0x5557e7620408 0x0~2000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate 0xa000/1000,a000,0 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7669a58 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9b58900 space 0x5557e7669a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x4e000~a000/1000,a000,0 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a28018 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9b58990 space 0x5557e7a28018 0x0~1000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x4e000~a000] 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11500 done 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c70a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e76204d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11500 [] 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 kv_submitted 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c70a0 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76204d0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11b00 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 finishing 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11b00 onodes 0x5557e9bc0500 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621048 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b58b40 space 0x5557e7621048 0x0~2000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e75c70a0) get_ref 0x0~2000 Blob(0x5557e75c70a0 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76204d0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e75c70a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a28478 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b58bd0 space 0x5557e7a28478 0x0~2000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11b00 done 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): adding buffer_update 0,147456 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e75c70a0 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76204d0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11b00 [] 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e76204e8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 0x5557e9bc0f00 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621198 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58d80 space 0x5557e7621198 0x0~2000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7668000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b57358 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58ea0 space 0x5557e9b57358 0x0~2000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668070 blob([0x50000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668000 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b57ba8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58f30 space 0x5557e9b57ba8 0x0~1000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b57eb8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58e10 space 0x5557e9b57eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7668070) get_ref 0x0~2000 Blob(0x5557e7668070 blob([0x50000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668000 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7668070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_append: {0~147456(147456)} 2019-12-07T19:33:58.633+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7668070 blob([0x50000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7668000 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e7668018 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): appending 0~147456 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7668230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668150 blob([0x52000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668230 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7668150) get_ref 0x0~2000 Blob(0x5557e7668150 blob([0x52000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668230 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7668150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e7668150 blob([0x52000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7668230 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e7668248 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=2, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cac00 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7620bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7620d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7620bd0 blob([0x54000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620d90 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7620bd0) get_ref 0x0~2000 Blob(0x5557e7620bd0 blob([0x54000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620d90 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7620bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e7620bd0 blob([0x54000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620d90 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e7620da8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7620c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7620e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7620c40 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620e00 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7620c40) get_ref 0x0~2000 Blob(0x5557e7620c40 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620e00 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7620c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e7620c40 blob([0x56000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620e00 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e7620e18 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0xa000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9bc1788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 = 0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._epoch' <- _epoch 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665200 cost 67390 (6 ios * 4000 + 43390 bytes) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665200 onodes 0x5557e9bc1680 shared_blobs 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9bc1788) update 1#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9bc1788) update inline shard 98 bytes from 5 extents 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:ecc72bdb:::smithi13140796-10:head# is 517 (413 bytes onode + 2 bytes spanning blobs + 102 bytes inline extents) 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665200 allocated 0x[4e000~a000] released 0x[] 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 freelist allocate 0x4e000~a000 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 prepare 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7665200 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e7683a00 con 0x5557e79cac00 2019-12-07T19:33:58.633+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=0, last_complete=20'1, committed=1, applied=1)) v2 -- 0x5557e9b36c00 con 0x5557e79ca800 2019-12-07T19:33:58.633+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e767c000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e75c4c60 prio 127 cost 0 latency 0.000586 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] add_log_entry 21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7a93b00 seq 6 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.9s5_head 0x5557e75e2b40) get_onode oid 5#1:9b8adabe:::smithi13140796-6:head# key 0x8580000000000000019b8adabe21736d'ithi13140796-6!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.633+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 32 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e62b8c00 con 0x5557e79cac00 2019-12-07T19:33:58.633+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.633+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4dc0 prio 196 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1091 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# = 0 2019-12-07T19:33:58.633+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~6000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.633+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 33 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e74f0000 con 0x5557e79cac00 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a93b00 3 blobs 2019-12-07T19:33:58.633+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4f20 prio 196 cost 0 latency 0.000009 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.633+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 fbmap_alloc 0x5557e6282e00 allocate 0x6000/1000,6000,0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x58000~6000/1000,6000,0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x58000~6000] 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c7dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c7dc0 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56fc0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.blob(0x5557e75c7dc0) get_ref 0x0~2000 Blob(0x5557e75c7dc0 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56fc0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.blob(0x5557e75c7dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e75c7dc0 blob([0x58000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b56fc0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e9b56fd8 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7620cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7620380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7620cb0 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620380 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7620cb0) get_ref 0x0~2000 Blob(0x5557e7620cb0 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620380 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7620cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7620cb0 blob([0x5a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620380 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e7620398 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76208c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76208c0 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621260 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.blob(0x5557e76208c0) get_ref 0x0~2000 Blob(0x5557e76208c0 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621260 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.blob(0x5557e76208c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e76208c0 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621260 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e7621278 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] encode_and_write: 1:3f7a742d:::smithi13140796-9:head new_size 147456 2019-12-07T19:33:58.633+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 19 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 34736+0+0 (crc 0 0 0) 0x5557e79e5000 con 0x5557e79ca800 2019-12-07T19:33:58.633+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x6000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e9bc1a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 = 0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.633+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 5 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e765f200 con 0x5557e79ca000 2019-12-07T19:33:58.633+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:3f7a742d:::smithi13140796-9:head resetting hinfo to logical size 147456 2019-12-07T19:33:58.633+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5080 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.633+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93b00 cost 42195 (4 ios * 4000 + 26195 bytes) 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): marking append 0 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93b00 onodes 0x5557e9bc1900 shared_blobs 2019-12-07T19:33:58.633+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 6 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e70e2800 con 0x5557e79ca000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e9bc1a08) update 5#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.633+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c51e0 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e9bc1a08) update inline shard 60 bytes from 3 extents 2019-12-07T19:33:58.633+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:9b8adabe:::smithi13140796-6:head# is 478 (412 bytes onode + 2 bytes spanning blobs + 64 bytes inline extents) 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93b00 allocated 0x[58000~6000] released 0x[] 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 freelist allocate 0x58000~6000 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~32768:2) Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(0~32768:1) Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:3f7a742d:::smithi13140796-9:head={0~147456(147456)}} 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 prepare 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a93b00 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:3f7a742d:::smithi13140796-9:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:25 client_op=osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~147456]}) 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:3f7a742d:::smithi13140796-9:head=[0~147456]} 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000171 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.6s1( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.633+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:3f7a742d:::smithi13140796-9:head,{0~147456(147456)} 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 20'1 (0'0,20'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:22, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 20'1 (0'0,20'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,20'1], crt=0'0) [21'2 (0'0) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:22 2019-12-07T19:33:58.629655+0000 0 ObjectCleanRegions clean_offsets: [131072~18446744073709420543], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=20'1 crt=0'0 lcod 20'1 active mbc={}] add_log_entry 21'2 (0'0) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:22 2019-12-07T19:33:58.629655+0000 0 ObjectCleanRegions clean_offsets: [131072~18446744073709420543], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=20'1 crt=0'0 lcod 20'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.633+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:25, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e9b9e800 con 0x5557e79cac00 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=20'1 crt=0'0 lcod 20'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.633+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4b00 finish 2019-12-07T19:33:58.633+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e75c4c60 finish 2019-12-07T19:33:58.633+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.633+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:25, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e9b9f000 con 0x5557e7519800 2019-12-07T19:33:58.633+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:25, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e9b9f800 con 0x5557e79ca800 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.633+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:25, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e763e800 con 0x5557e79cbc00 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9b11800 seq 6 2019-12-07T19:33:58.633+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:25, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x5557e763f800 con 0x5557e79ca000 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) get_onode oid 1#1:6c205177:::smithi13140796-8:head# key 0x8180000000000000016c2051'w!smithi13140796-8!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.633+0000 7f2b744da700 20 earliest_dup_version = 0 2019-12-07T19:33:58.633+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1092 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# = 0 2019-12-07T19:33:58.633+0000 7f2b744da700 20 trim 21'3 (0'0) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:25 2019-12-07T19:33:58.630729+0000 0 ObjectCleanRegions clean_offsets: [147456~18446744073709404159], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.633+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 2 keys 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 2 keys = 0 2019-12-07T19:33:58.633+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.633+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 0x0~8000 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:6c205177:::smithi13140796-8:head# 0x0~8000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~8000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.633+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log log((0'0,21'2], crt=0'0) [21'3 (0'0) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:25 2019-12-07T19:33:58.630729+0000 0 ObjectCleanRegions clean_offsets: [147456~18446744073709404159], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b11800 4 blobs 2019-12-07T19:33:58.633+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'3 (0'0) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:25 2019-12-07T19:33:58.630729+0000 0 ObjectCleanRegions clean_offsets: [147456~18446744073709404159], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 3 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.633+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x8000/1000,8000,0 2019-12-07T19:33:58.633+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x5e000~8000/1000,8000,0 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x5e000~8000] 2019-12-07T19:33:58.633+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76210a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76210a0 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2e70 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore.blob(0x5557e76210a0) get_ref 0x0~2000 Blob(0x5557e76210a0 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2e70 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore.blob(0x5557e76210a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e76210a0 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2e70 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9be2e88 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7630f00 seq 8 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2ee0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2f50 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be2ee0) get_ref 0x0~2000 Blob(0x5557e9be2ee0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2f50 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be2ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3f7a742d:::smithi13140796-9:head# key 0x8080000000000000013f7a74'-!smithi13140796-9!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9be2ee0 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2f50 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9be2f68 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.634+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1093 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# = 0 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2fc0 blob([0x62000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3030 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be2fc0) get_ref 0x0~2000 Blob(0x5557e9be2fc0 blob([0x62000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3030 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be2fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.634+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2 keys 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9be2fc0 blob([0x62000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be3030 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2 keys = 0 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9be3048 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.634+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be30a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.634+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 0x0~9000 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be30a0 blob([0x64000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3110 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 0#1:3f7a742d:::smithi13140796-9:head# 0x0~9000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be30a0) get_ref 0x0~2000 Blob(0x5557e9be30a0 blob([0x64000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3110 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be30a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~9000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9be30a0 blob([0x64000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be3110 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9be3128 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7630f00 5 blobs 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x8000 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0888) dirty_range mark inline shard dirty 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 0x0~8000 = 0 2019-12-07T19:33:58.634+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.634+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11800 cost 55177 (5 ios * 4000 + 35177 bytes) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11800 onodes 0x5557e9bc0780 shared_blobs 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0888) update 1#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:33:58.634+0000 7f2b744da700 10 fbmap_alloc 0x5557e6282e00 allocate 0x9000/1000,9000,0 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0888) update inline shard 79 bytes from 4 extents 2019-12-07T19:33:58.634+0000 7f2b744da700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x66000~9000/1000,9000,0 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x66000~9000] 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:6c205177:::smithi13140796-8:head# is 497 (412 bytes onode + 2 bytes spanning blobs + 83 bytes inline extents) 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11800 allocated 0x[5e000~8000] released 0x[] 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7621570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 freelist allocate 0x5e000~8000 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7621570 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621500 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e7621570) get_ref 0x0~2000 Blob(0x5557e7621570 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621500 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 prepare 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e7621570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b11800 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7621570 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621500 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621518 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76215e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76215e0 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621650 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e76215e0) get_ref 0x0~2000 Blob(0x5557e76215e0 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621650 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e76215e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e76215e0 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621650 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621668 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7621b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e76219d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7621b20 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76219d0 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e7621b20) get_ref 0x0~2000 Blob(0x5557e7621b20 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76219d0 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e7621b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e7621b20 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76219d0 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e76219e8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7621c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7621c00 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621a40 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e7621c00) get_ref 0x0~2000 Blob(0x5557e7621c00 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621a40 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e7621c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e7621c00 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621a40 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621a58 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7621ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7621ab0 blob([0x6e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2d20 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e7621ab0) get_ref 0x0~1000 Blob(0x5557e7621ab0 blob([0x6e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2d20 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.blob(0x5557e7621ab0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~1000: 0x0~1000 Blob(0x5557e7621ab0 blob([0x6e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9be2d20 sbid 0x0)) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9be2d38 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x9000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.extentmap(0x5557e9c7c388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 0x0~9000 = 0 2019-12-07T19:33:58.634+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=20'1 crt=0'0 lcod 20'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.634+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.634+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7694b00 finish 2019-12-07T19:33:58.634+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630f00 cost 78483 (10 ios * 4000 + 38483 bytes) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630f00 onodes 0x5557e9c7c280 shared_blobs 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.extentmap(0x5557e9c7c388) update 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore.extentmap(0x5557e9c7c388) update inline shard 95 bytes from 5 extents 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:3f7a742d:::smithi13140796-9:head# is 513 (412 bytes onode + 2 bytes spanning blobs + 99 bytes inline extents) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630f00 allocated 0x[66000~9000] released 0x[] 2019-12-07T19:33:58.634+0000 7f2b744da700 10 freelist allocate 0x66000~9000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 prepare 2019-12-07T19:33:58.634+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7630f00 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'3 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x5557e7967b00 21'3 rep_tid=2 committed?=0 r=0) 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4420 finish 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.634+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 aio_wait 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4dc0 prio 196 cost 0 latency 0.001004 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.634+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630f00 2019-12-07T19:33:58.634+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 io_done 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4dc0 finish 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4f20 prio 196 cost 0 latency 0.001012 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4f20 finish 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5080 prio 196 cost 0 latency 0.000892 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5080 finish 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:26 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 62 e21) queued 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c51e0 prio 196 cost 0 latency 0.000905 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c51e0 finish 2019-12-07T19:33:58.634+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 10 ==== MOSDECSubOpWriteReply(1.cs0 19/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e7a9b200 con 0x5557e7519800 2019-12-07T19:33:58.634+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695340 prio 196 cost 0 latency 0.000012 epoch 19 MOSDECSubOpWriteReply(1.cs0 19/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 19/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e19) 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:26 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 62 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c46e0 prio 63 cost 62 latency 0.002343 osd_op(client.4540.0:26 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:26 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x5557e9b21680 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(excl n=1 w=0) oi: 1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.634+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c46e0 finish 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 19/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e19) queued 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c49a0 prio 63 cost 0 latency 0.002450 osd_op(client.4540.0:27 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:27 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x5557e9b21680 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(excl n=1 w=1) oi: 1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.634+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1223 < min_flush_size 524288 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7c280 had 1 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.634+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1246 < min_flush_size 524288 2019-12-07T19:33:58.634+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1246 < min_flush_size 524288 2019-12-07T19:33:58.634+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.634+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2479a~4de to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.634+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2479a 0x2479a~4de to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.634+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x479a 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x79a 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.634+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xc78 and padding block with 0x388 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.634+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c49a0 finish 2019-12-07T19:33:58.634+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.634+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 19/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e19) pg 0x5557e6e89000 2019-12-07T19:33:58.634+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695340 prio 196 cost 0 latency 0.000351 MOSDECSubOpWriteReply(1.cs0 19/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.635+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 aio_wait 2019-12-07T19:33:58.635+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665200 2019-12-07T19:33:58.635+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 io_done 2019-12-07T19:33:58.635+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 aio_wait 2019-12-07T19:33:58.635+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93b00 2019-12-07T19:33:58.635+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 io_done 2019-12-07T19:33:58.635+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 19/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.635+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 aio_wait 2019-12-07T19:33:58.635+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11800 2019-12-07T19:33:58.635+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695340 finish 2019-12-07T19:33:58.635+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 io_done 2019-12-07T19:33:58.635+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.635+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x24c78 2019-12-07T19:33:58.635+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.635+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.635+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.635+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.001816642s (0.000358086s flush + 0.001458556s kv commit) 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=3 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630f00> 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 kv_submitted 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630f00 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2043 < min_flush_size 524288 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630f00 finishing 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630f00 onodes 0x5557e9c7c280 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621518 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9b59200 space 0x5557e7621518 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621668 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9b59290 space 0x5557e7621668 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e76219e8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9b59320 space 0x5557e76219e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621a58 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9b59b00 space 0x5557e7621a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be2d38 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9b59b90 space 0x5557e9be2d38 0x0~1000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc1680 had 1 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630f00 done 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630f00 [] 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3231 < min_flush_size 524288 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc1900 had 1 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5228 < min_flush_size 524288 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0780 had 1 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5251 < min_flush_size 524288 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5251 < min_flush_size 524288 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x24c78~1483 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x24c78 0x24c78~1483 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x4c78 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xc78 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xfb and padding block with 0xf05 2019-12-07T19:33:58.636+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x260fb 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.636+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 3 cleaned 0 in 0.000159785s (0.000004312s flush + 0.000155473s kv commit) 2019-12-07T19:33:58.636+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7665200,0x5557e7a93b00,0x5557e9b11800> 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 kv_submitted 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665200 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 finishing 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665200 onodes 0x5557e9bc1680 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e76204e8 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aac60 space 0x5557e76204e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7620da8 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aa990 space 0x5557e7620da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7620e18 in 0x5557e6269880) _finish_write discard buffer(0x5557e79ab170 space 0x5557e7620e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7668018 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aa360 space 0x5557e7668018 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7668248 in 0x5557e6269880) _finish_write discard buffer(0x5557e79ab950 space 0x5557e7668248 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665200 done 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665200 [] 2019-12-07T19:33:58.636+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 kv_submitted 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93b00 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 finishing 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93b00 onodes 0x5557e9bc1900 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7620398 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b58630 space 0x5557e7620398 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621278 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b58870 space 0x5557e7621278 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=3, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e76b1a00 con 0x5557e79cac00 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56fd8 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b587e0 space 0x5557e9b56fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93b00 done 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93b00 [] 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 kv_submitted 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11800 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 finishing 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11800 onodes 0x5557e9bc0780 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be2e88 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b594d0 space 0x5557e9be2e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=4, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e7a9b600 con 0x5557e79cac00 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be2f68 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b59560 space 0x5557e9be2f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be3048 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b595f0 space 0x5557e9be3048 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be3128 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b59680 space 0x5557e9be3128 0x0~2000 writing nocache) 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11800 done 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11800 [] 2019-12-07T19:33:58.636+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.636+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9b37400 con 0x5557e79ca800 2019-12-07T19:33:58.636+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.636+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.636+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.637+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 14 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c76200 con 0x5557e79cbc00 2019-12-07T19:33:58.637+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5340 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.637+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.637+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.637+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.637+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.637+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.637+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5340 prio 196 cost 0 latency 0.000042 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.637+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.637+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5340 finish 2019-12-07T19:33:58.637+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.638+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 7 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9b37400 con 0x5557e79ca000 2019-12-07T19:33:58.638+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c54a0 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.638+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.638+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 11 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9b37000 con 0x5557e7519800 2019-12-07T19:33:58.638+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695ce0 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.638+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.638+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c54a0 prio 196 cost 0 latency 0.000046 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.638+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.638+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.638+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.638+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c54a0 finish 2019-12-07T19:33:58.638+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.638+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.638+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695ce0 prio 196 cost 0 latency 0.000054 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.638+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.638+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695ce0 finish 2019-12-07T19:33:58.638+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 12 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e765e200 con 0x5557e7519800 2019-12-07T19:33:58.638+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.638+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695a20 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.638+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.638+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695a20 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.638+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.638+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695a20 finish 2019-12-07T19:33:58.638+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.639+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 34 ==== MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2228+0+0 (unknown 0 0 0) 0x5557e7a0e000 con 0x5557e79cac00 2019-12-07T19:33:58.639+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5600 prio 127 cost 0 latency 0.000064 epoch 21 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.639+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e767c000 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e75c5600 prio 127 cost 0 latency 0.000140 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.9s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'2], crt=0'0) [21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.639+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 35 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:31, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 47028+0+0 (unknown 0 0 0) 0x5557e9b9e800 con 0x5557e79cac00 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'2 active mbc={}] add_log_entry 21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.639+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5760 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:31, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.639+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:31, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 osd.6 pg_epoch: 21 pg[1.9s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.639+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7121800 seq 7 2019-12-07T19:33:58.639+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# 3 keys 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# 3 keys = 0 2019-12-07T19:33:58.639+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:31, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.639+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:31, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e75c5760 prio 127 cost 0 latency 0.000131 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:31, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.8s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121800 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121800 onodes 0x5557e76cb900 shared_blobs 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e76cba08) update 5#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:9694a0e5:::smithi13140796-3:head# is 526 (482 bytes onode + 2 bytes spanning blobs + 42 bytes inline extents) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:31, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 prepare 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121800 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 io_done 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [21'2 (0'0) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:31 2019-12-07T19:33:58.635123+0000 0 ObjectCleanRegions clean_offsets: [180224~18446744073709371391], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] add_log_entry 21'2 (0'0) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:31 2019-12-07T19:33:58.635123+0000 0 ObjectCleanRegions clean_offsets: [180224~18446744073709371391], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.639+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e75c5600 finish 2019-12-07T19:33:58.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7692900 seq 6 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:1353159e:::smithi13140796-11:head# key 0x8580000000000000011353159e21736d'ithi13140796-11!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1094 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# = 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 2 keys 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 2 keys = 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~b000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692900 6 blobs 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 fbmap_alloc 0x5557e6282e00 allocate 0xb000/1000,b000,0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x6f000~b000/1000,b000,0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x6f000~b000] 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7620fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2e00 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620fc0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9be2e00) get_ref 0x0~2000 Blob(0x5557e9be2e00 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620fc0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9be2e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9be2e00 blob([0x6f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620fc0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e7620fd8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be31f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2d90 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be31f0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9be2d90) get_ref 0x0~2000 Blob(0x5557e9be2d90 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be31f0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9be2d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9be2d90 blob([0x71000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be31f0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9be3208 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c7110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c7110 blob([0x73000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6b60 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e75c7110) get_ref 0x0~2000 Blob(0x5557e75c7110 blob([0x73000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6b60 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e75c7110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e75c7110 blob([0x73000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c6b60 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e75c6b78 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b572d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6a10 blob([0x75000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b572d0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e75c6a10) get_ref 0x0~2000 Blob(0x5557e75c6a10 blob([0x75000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b572d0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e75c6a10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e75c6a10 blob([0x75000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b572d0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9b572e8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be3260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be32d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be3260 blob([0x77000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be32d0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9be3260) get_ref 0x0~2000 Blob(0x5557e9be3260 blob([0x77000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be32d0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9be3260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9be3260 blob([0x77000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be32d0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9be32e8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be3340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be33b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be3340 blob([0x79000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be33b0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9be3340) get_ref 0x0~1000 Blob(0x5557e9be3340 blob([0x79000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be33b0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9be3340) get_ref init 0x1000, 1000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~1000: 0x0~1000 Blob(0x5557e9be3340 blob([0x79000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9be33b0 sbid 0x0)) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9be33c8 in 0x5557e6269960) _discard 0x0~1000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0xb000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9c7d288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 = 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692900 cost 74678 (7 ios * 4000 + 46678 bytes) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692900 onodes 0x5557e9c7d180 shared_blobs 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9c7d288) update 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9c7d288) update inline shard 114 bytes from 6 extents 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:1353159e:::smithi13140796-11:head# is 533 (413 bytes onode + 2 bytes spanning blobs + 118 bytes inline extents) 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692900 allocated 0x[6f000~b000] released 0x[] 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 freelist allocate 0x6f000~b000 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 prepare 2019-12-07T19:33:58.639+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692900 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.639+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1563 < min_flush_size 524288 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb900 had 1 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:58.639+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1586 < min_flush_size 524288 2019-12-07T19:33:58.639+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1586 < min_flush_size 524288 2019-12-07T19:33:58.639+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.639+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x260fb~632 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.639+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x260fb 0x260fb~632 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x60fb 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xfb 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.639+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x72d and padding block with 0x8d3 2019-12-07T19:33:58.639+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 36 ==== MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 55220+0+0 (unknown 0 0 0) 0x5557e7a10000 con 0x5557e79cac00 2019-12-07T19:33:58.640+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000560 epoch 21 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.640+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e767c000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000622 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.9s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] append_log log((0'0,21'3], crt=21'1) [21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'2 active mbc={}] add_log_entry 21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.640+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'2 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.640+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e75c5760 finish 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.640+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7692f00 seq 8 2019-12-07T19:33:58.640+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 aio_wait 2019-12-07T19:33:58.640+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692900 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.9s5_head 0x5557e75e2b40) get_onode oid 5#1:9da56077:::smithi13140796-13:head# key 0x8580000000000000019da560'w!smithi13140796-13!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.640+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 io_done 2019-12-07T19:33:58.640+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1095 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# = 0 2019-12-07T19:33:58.640+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 13 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:34, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 51124+0+0 (unknown 0 0 0) 0x5557e9b9f000 con 0x5557e7519800 2019-12-07T19:33:58.640+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# 2 keys 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# 2 keys = 0 2019-12-07T19:33:58.640+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c4580 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:34, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.640+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.640+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:34, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# 0x0~d000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:9da56077:::smithi13140796-13:head# 0x0~d000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~d000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692f00 7 blobs 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:34, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:34, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 osd.6 21 dequeue_op 0x5557e75c4580 prio 127 cost 0 latency 0.000073 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:34, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:34, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 fbmap_alloc 0x5557e6282e00 allocate 0xd000/1000,d000,0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x7a000~d000/1000,d000,0 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:34 2019-12-07T19:33:58.636620+0000 0 ObjectCleanRegions clean_offsets: [196608~18446744073709355007], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x7a000~d000] 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669880 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621880 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e7669880) get_ref 0x0~2000 Blob(0x5557e7669880 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621880 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e7669880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:34 2019-12-07T19:33:58.636620+0000 0 ObjectCleanRegions clean_offsets: [196608~18446744073709355007], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7669880 blob([0x7a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621880 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.BufferSpace(0x5557e7621898 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7669960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669810 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669960 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e7669810) get_ref 0x0~2000 Blob(0x5557e7669810 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669960 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e7669810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7669810 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7669960 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.BufferSpace(0x5557e7669978 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb2000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb2000 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2070 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb2000) get_ref 0x0~2000 Blob(0x5557e9cb2000 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2070 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb2000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9cb2000 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2070 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.BufferSpace(0x5557e9cb2088 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7692000 seq 5 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb20e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb20e0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2150 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb20e0) get_ref 0x0~2000 Blob(0x5557e9cb20e0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2150 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb20e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.5s1_head 0x5557e6281c20) get_onode oid 1#1:a9dea763:::smithi13140796-12:head# key 0x818000000000000001a9dea7'c!smithi13140796-12!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9cb20e0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2150 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.BufferSpace(0x5557e9cb2168 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1096 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb21c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# = 0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb21c0 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2230 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 2 keys 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb21c0) get_ref 0x0~2000 Blob(0x5557e9cb21c0 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2230 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb21c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9cb21c0 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2230 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 2 keys = 0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.BufferSpace(0x5557e9cb2248 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb22a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb22a0 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2310 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 0x0~c000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb22a0) get_ref 0x0~2000 Blob(0x5557e9cb22a0 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2310 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb22a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:a9dea763:::smithi13140796-12:head# 0x0~c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9cb22a0 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2310 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.BufferSpace(0x5557e9cb2328 in 0x5557e6269a40) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~c000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb2380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb23f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb2380 blob([0x86000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb23f0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb2380) get_ref 0x0~1000 Blob(0x5557e9cb2380 blob([0x86000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb23f0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.blob(0x5557e9cb2380) get_ref init 0x1000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692000 6 blobs 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~1000: 0x0~1000 Blob(0x5557e9cb2380 blob([0x86000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9cb23f0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.BufferSpace(0x5557e9cb2408 in 0x5557e6269a40) _discard 0x0~1000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0xd000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.extentmap(0x5557e9b7c108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# 0x0~d000 = 0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 fbmap_alloc 0x5557e6282e00 allocate 0xc000/1000,c000,0 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x87000~c000/1000,c000,0 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x87000~c000] 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb25b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692f00 cost 86870 (8 ios * 4000 + 54870 bytes) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692f00 onodes 0x5557e9b7c000 shared_blobs 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6ee0 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb25b0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.extentmap(0x5557e9b7c108) update 5#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e75c6ee0) get_ref 0x0~2000 Blob(0x5557e75c6ee0 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb25b0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e75c6ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e75c6ee0 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb25b0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9cb25c8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore.extentmap(0x5557e9b7c108) update inline shard 133 bytes from 7 extents 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:9da56077:::smithi13140796-13:head# is 552 (413 bytes onode + 2 bytes spanning blobs + 137 bytes inline extents) 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692f00 allocated 0x[7a000~d000] released 0x[] 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 freelist allocate 0x7a000~d000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 freelist _xor first_key 0x0 last_key 0x80000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb28c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 prepare 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692f00 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb28c0 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2930 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb28c0) get_ref 0x0~2000 Blob(0x5557e9cb28c0 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2930 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb28c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9cb28c0 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2930 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9cb2948 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb29a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb29a0 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2a10 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb29a0) get_ref 0x0~2000 Blob(0x5557e9cb29a0 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2a10 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb29a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9cb29a0 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2a10 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9cb2a28 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb2a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb2a80 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2af0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb2a80) get_ref 0x0~2000 Blob(0x5557e9cb2a80 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2af0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb2a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9cb2a80 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2af0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9cb2b08 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb2b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb2b60 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2bd0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb2b60) get_ref 0x0~2000 Blob(0x5557e9cb2b60 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2bd0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb2b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9cb2b60 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2bd0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9cb2be8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb2c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb2c40 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2cb0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb2c40) get_ref 0x0~2000 Blob(0x5557e9cb2c40 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb2cb0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9cb2c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9cb2c40 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb2cb0 sbid 0x0)) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9cb2cc8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0xc000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557e9b7cb08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 0x0~c000 = 0 2019-12-07T19:33:58.640+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.640+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 15 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36c00 con 0x5557e79cbc00 2019-12-07T19:33:58.640+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.640+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5a20 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.640+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692000 cost 79582 (7 ios * 4000 + 51582 bytes) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692000 onodes 0x5557e9b7ca00 shared_blobs 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557e9b7cb08) update 1#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:33:58.640+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 16 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e7683a00 con 0x5557e79cbc00 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557e9b7cb08) update inline shard 117 bytes from 6 extents 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4c60 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:a9dea763:::smithi13140796-12:head# is 536 (413 bytes onode + 2 bytes spanning blobs + 121 bytes inline extents) 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.640+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692000 allocated 0x[87000~c000] released 0x[] 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'2 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 freelist allocate 0x87000~c000 2019-12-07T19:33:58.640+0000 7f2b77ce1700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5a20 prio 196 cost 0 latency 0.000053 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 prepare 2019-12-07T19:33:58.640+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e75c58c0 finish 2019-12-07T19:33:58.640+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692000 2019-12-07T19:33:58.640+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5a20 finish 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.640+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 37 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e76b0000 con 0x5557e79cac00 2019-12-07T19:33:58.640+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c54a0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.640+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4c60 prio 196 cost 0 latency 0.000061 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4c60 finish 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.640+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 38 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:20, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2228+0+0 (unknown 0 0 0) 0x5557e7a11000 con 0x5557e79cac00 2019-12-07T19:33:58.640+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:20, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c54a0 prio 196 cost 0 latency 0.000038 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.640+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:20, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.640+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c54a0 finish 2019-12-07T19:33:58.640+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.640+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:20, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.640+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:20, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.640+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 aio_wait 2019-12-07T19:33:58.640+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2672d 2019-12-07T19:33:58.640+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:20, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.8s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 lcod 21'1 active mbc={}] 2019-12-07T19:33:58.640+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692f00 2019-12-07T19:33:58.640+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.640+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.640+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.640+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.640+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 lcod 21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:20, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.640+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 io_done 2019-12-07T19:33:58.640+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 lcod 21'1 active mbc={}] append_log log((0'0,21'2], crt=0'0) [21'3 (21'1) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:20 2019-12-07T19:33:58.628837+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.640+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'1 active mbc={}] add_log_entry 21'3 (21'1) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:20 2019-12-07T19:33:58.628837+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.640+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:19 2019-12-07T19:33:58.628809+0000 0 ObjectCleanRegions clean_offsets: [114688~18446744073709436927], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.640+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.640+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.640+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.640+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.640+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7693200 seq 7 2019-12-07T19:33:58.640+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 3 keys 2019-12-07T19:33:58.640+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 3 keys = 0 2019-12-07T19:33:58.640+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.640+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.640+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.640+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.640+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693200 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693200 onodes 0x5557e9bc0f00 shared_blobs 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore.extentmap(0x5557e9bc1008) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 565 (483 bytes onode + 2 bytes spanning blobs + 80 bytes inline extents) 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.640+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 prepare 2019-12-07T19:33:58.640+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693200 2019-12-07T19:33:58.640+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 io_done 2019-12-07T19:33:58.640+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.640+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4f20 finish 2019-12-07T19:33:58.640+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.641+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.641+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 20 ==== MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 59316+0+0 (crc 0 0 0) 0x5557e7a11800 con 0x5557e79ca800 2019-12-07T19:33:58.641+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c51e0 prio 127 cost 0 latency 0.000036 epoch 21 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.641+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.641+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.641+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.641+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.641+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e769f000 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e75c51e0 prio 127 cost 0 latency 0.000080 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:40 2019-12-07T19:33:58.639869+0000 0 ObjectCleanRegions clean_offsets: [229376~18446744073709322239], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:40 2019-12-07T19:33:58.639869+0000 0 ObjectCleanRegions clean_offsets: [229376~18446744073709322239], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.641+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:58.641+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e7121500 seq 5 2019-12-07T19:33:58.641+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.1s2_head 0x5557e76a8000) get_onode oid 2#1:891f0d13:::smithi13140796-14:head# key 0x828000000000000001891f0d1321736d'ithi13140796-14!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.641+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.641+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1097 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# = 0 2019-12-07T19:33:58.641+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 2 keys 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 2 keys = 0 2019-12-07T19:33:58.641+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.641+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.642+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 0x0~e000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 2#1:891f0d13:::smithi13140796-14:head# 0x0~e000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~e000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 7 blobs 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 fbmap_alloc 0x5557e6282e00 allocate 0xe000/1000,e000,0 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x93000~e000/1000,e000,0 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x93000~e000] 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b570a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b570a0 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3110 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e9b570a0) get_ref 0x0~2000 Blob(0x5557e9b570a0 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3110 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e9b570a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9b570a0 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3110 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e9cb3128 in 0x5557e6269500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7621810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7621810 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57490 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 39 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x5557e9b9e000 con 0x5557e79cac00 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7621810) get_ref 0x0~2000 Blob(0x5557e7621810 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57490 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7621810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7621810 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57490 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4dc0 prio 127 cost 0 latency 0.000036 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e9b574a8 in 0x5557e6269500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a28e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a28e70 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29810 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7a28e70) get_ref 0x0~2000 Blob(0x5557e7a28e70 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29810 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7a28e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e7a28e70 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7a29810 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e7a29828 in 0x5557e6269500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a29180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a29180 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28c40 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7a29180) get_ref 0x0~2000 Blob(0x5557e7a29180 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a28c40 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7a29180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e7a29180 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7a28c40 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e7a28c58 in 0x5557e6269500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a28770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e76cd960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a28770 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76cd960 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7a28770) get_ref 0x0~2000 Blob(0x5557e7a28770 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76cd960 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e7a28770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e7a28770 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76cd960 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e76cd978 in 0x5557e6269500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4dc0 prio 127 cost 0 latency 0.000086 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.642+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e9cc4000 seq 6 2019-12-07T19:33:58.642+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76cd9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e76cd260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76cd9d0 blob([0x9d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76cd260 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e76cd9d0) get_ref 0x0~2000 Blob(0x5557e76cd9d0 blob([0x9d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76cd260 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e76cd9d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4000 onodes shared_blobs 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e76cd9d0 blob([0x9d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76cd260 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e76cd278 in 0x5557e6269500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 prepare 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4000 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 io_done 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb38f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4dc0 finish 2019-12-07T19:33:58.642+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb38f0 blob([0x9f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3960 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e9cb38f0) get_ref 0x0~2000 Blob(0x5557e9cb38f0 blob([0x9f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3960 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.blob(0x5557e9cb38f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9cb38f0 blob([0x9f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3960 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e9cb3978 in 0x5557e6269500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0xe000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e76cb788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 0x0~e000 = 0 2019-12-07T19:33:58.642+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.642+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._epoch' <- _epoch 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121500 cost 91774 (8 ios * 4000 + 59774 bytes) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121500 onodes 0x5557e76cb680 shared_blobs 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e76cb788) update 2#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e76cb788) update inline shard 136 bytes from 7 extents 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 2#1:891f0d13:::smithi13140796-14:head# is 555 (413 bytes onode + 2 bytes spanning blobs + 140 bytes inline extents) 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121500 allocated 0x[93000~e000] released 0x[] 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 freelist allocate 0x93000~e000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 prepare 2019-12-07T19:33:58.642+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7121500 2019-12-07T19:33:58.642+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 40 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x5557e9ba1000 con 0x5557e79cac00 2019-12-07T19:33:58.642+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5080 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c5080 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.642+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7665500 seq 7 2019-12-07T19:33:58.642+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 3 keys 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 3 keys = 0 2019-12-07T19:33:58.642+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.642+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.642+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665500 onodes 0x5557e9bc1680 shared_blobs 2019-12-07T19:33:58.642+0000 7f2b76cdf700 20 bluestore.extentmap(0x5557e9bc1788) update 1#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.643+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:ecc72bdb:::smithi13140796-10:head# is 589 (485 bytes onode + 2 bytes spanning blobs + 102 bytes inline extents) 2019-12-07T19:33:58.643+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.643+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 prepare 2019-12-07T19:33:58.643+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665500 2019-12-07T19:33:58.643+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 io_done 2019-12-07T19:33:58.643+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.643+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c5080 finish 2019-12-07T19:33:58.643+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.643+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 41 ==== MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 2228+0+0 (unknown 0 0 0) 0x5557e9ba1800 con 0x5557e79cac00 2019-12-07T19:33:58.643+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.643+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e767c000 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.9s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] append_log log((0'0,21'4], crt=21'1) [21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'1 lcod 21'2 active mbc={}] add_log_entry 21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'2 active mbc={}] rollforward: entry=21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.643+0000 7f2b79ce5700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7693800 seq 9 2019-12-07T19:33:58.643+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# 3 keys 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# 3 keys = 0 2019-12-07T19:33:58.643+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.643+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693800 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693800 onodes 0x5557e9bc1900 shared_blobs 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore.extentmap(0x5557e9bc1a08) update 5#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:9b8adabe:::smithi13140796-6:head# is 549 (483 bytes onode + 2 bytes spanning blobs + 64 bytes inline extents) 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 prepare 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693800 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 io_done 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.643+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e75c5b80 finish 2019-12-07T19:33:58.643+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.644+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 114809 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.644+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 42 ==== MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 63412+0+0 (unknown 0 0 0) 0x5557e9ba0000 con 0x5557e79cac00 2019-12-07T19:33:58.644+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5ce0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.644+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e766d000 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e75c5ce0 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:43 2019-12-07T19:33:58.641743+0000 0 ObjectCleanRegions clean_offsets: [245760~18446744073709305855], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:43 2019-12-07T19:33:58.641743+0000 0 ObjectCleanRegions clean_offsets: [245760~18446744073709305855], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.644+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e9cc5800 seq 5 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.as1_head 0x5557e75e2d20) get_onode oid 1#1:51ccf92f:::smithi13140796-15:head# key 0x81800000000000000151ccf9'/!smithi13140796-15!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.644+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1098 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# = 0 2019-12-07T19:33:58.644+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 2 keys 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 2 keys = 0 2019-12-07T19:33:58.644+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.644+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~f000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9cc5800 8 blobs 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate 0xf000/1000,f000,0 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xa1000~f000/1000,f000,0 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xa1000~f000] 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb32d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668620 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb32d0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7668620) get_ref 0x0~2000 Blob(0x5557e7668620 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb32d0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7668620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7668620 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb32d0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9cb32e8 in 0x5557e6269b20) _discard 0x0~2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7668310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668700 blob([0xa3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668310 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.004576137s (0.000000133s flush + 0.004576004s kv commit) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7668700) get_ref 0x0~2000 Blob(0x5557e7668700 blob([0xa3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668310 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7668700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.644+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:33:58.644+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7668700 blob([0xa3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7668310 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e7668328 in 0x5557e6269b20) _discard 0x0~2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76688c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7668ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76688c0 blob([0xa5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668ee0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e76688c0) get_ref 0x0~2000 Blob(0x5557e76688c0 blob([0xa5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668ee0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e76688c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e76688c0 blob([0xa5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7668ee0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e7668ef8 in 0x5557e6269b20) _discard 0x0~2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e76692d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669260 blob([0xa7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76692d0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7669260) get_ref 0x0~2000 Blob(0x5557e7669260 blob([0xa7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76692d0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7669260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.644+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121800> 2019-12-07T19:33:58.644+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e7669260 blob([0xa7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76692d0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 kv_submitted 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e76692e8 in 0x5557e6269b20) _discard 0x0~2000 2019-12-07T19:33:58.644+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121800 2019-12-07T19:33:58.644+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 finishing 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7669180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.644+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121800 onodes 0x5557e76cb900 2019-12-07T19:33:58.644+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121800 done 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669340 blob([0xa9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669180 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692f00 kv_queued 2019-12-07T19:33:58.644+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121800 [] 2019-12-07T19:33:58.644+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7669340) get_ref 0x0~2000 Blob(0x5557e7669340 blob([0xa9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669180 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7669340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e7669340 blob([0xa9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7669180 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e7669198 in 0x5557e6269b20) _discard 0x0~2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7669110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669030 blob([0xab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669110 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7669030) get_ref 0x0~2000 Blob(0x5557e7669030 blob([0xab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669110 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7669030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e7669030 blob([0xab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7669110 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e7669128 in 0x5557e6269b20) _discard 0x0~2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668f50 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7030 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7668f50) get_ref 0x0~2000 Blob(0x5557e7668f50 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7030 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e7668f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e7668f50 blob([0xad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7030 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e75c7048 in 0x5557e6269b20) _discard 0x0~2000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb3180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb3180 blob([0xaf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3ea0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e9cb3180) get_ref 0x0~1000 Blob(0x5557e9cb3180 blob([0xaf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3ea0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.blob(0x5557e9cb3180) get_ref init 0x1000, 1000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~1000: 0x0~1000 Blob(0x5557e9cb3180 blob([0xaf000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9cb3ea0 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9cb3eb8 in 0x5557e6269b20) _discard 0x0~1000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0xf000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9b7d788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 = 0 2019-12-07T19:33:58.644+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.644+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:58.644+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=5, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e9b36e00 con 0x5557e79cac00 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.644+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5800 cost 99870 (9 ios * 4000 + 63870 bytes) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5800 onodes 0x5557e9b7d680 shared_blobs 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9b7d788) update 1#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9b7d788) update inline shard 152 bytes from 8 extents 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:51ccf92f:::smithi13140796-15:head# is 571 (413 bytes onode + 2 bytes spanning blobs + 156 bytes inline extents) 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5800 allocated 0x[a1000~f000] released 0x[] 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 freelist allocate 0xa1000~f000 2019-12-07T19:33:58.644+0000 7f2b754dc700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 prepare 2019-12-07T19:33:58.644+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9cc5800 2019-12-07T19:33:58.645+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 21 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2 ==== 2228+0+0 (crc 0 0 0) 0x5557e9b9f800 con 0x5557e79ca800 2019-12-07T19:33:58.645+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2 2019-12-07T19:33:58.645+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2 pg pg[1.6s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:11, at_version=21'3, trim_to=0'0, roll_forward_to=20'1)) v2 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'2], crt=0'0) [21'3 (20'1) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:11 2019-12-07T19:33:58.626793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'2 active mbc={}] add_log_entry 21'3 (20'1) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:11 2019-12-07T19:33:58.626793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=20'1 lcod 21'2 active mbc={}] rollforward: entry=20'1 (0'0) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:10 2019-12-07T19:33:58.626763+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=20'1 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=20'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.645+0000 7f2b774e0700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9d5a000 seq 7 2019-12-07T19:33:58.645+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 3 keys 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 3 keys = 0 2019-12-07T19:33:58.645+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.645+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5a000 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5a000 onodes 0x5557e9bc0500 shared_blobs 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e9bc0608) update 1#1:657ac0fe:::smithi13140796-4:head# 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:657ac0fe:::smithi13140796-4:head# is 530 (483 bytes onode + 2 bytes spanning blobs + 45 bytes inline extents) 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5a000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a000 prepare 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5a000 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a000 io_done 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=20'1 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.645+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c5e40 finish 2019-12-07T19:33:58.645+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.646+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 43 ==== MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 909+0+0 (unknown 0 0 0) 0x5557e70a7000 con 0x5557e79cac00 2019-12-07T19:33:58.646+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4840 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.646+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e766a000 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4840 prio 127 cost 0 latency 0.000194 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:13 2019-12-07T19:33:58.627318+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.646+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e9b10900 seq 6 2019-12-07T19:33:58.646+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10900 onodes shared_blobs 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 prepare 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10900 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 io_done 2019-12-07T19:33:58.646+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4840 finish 2019-12-07T19:33:58.646+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.646+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 22 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36200 con 0x5557e79ca800 2019-12-07T19:33:58.646+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5340 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.646+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.646+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 23 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36000 con 0x5557e79ca800 2019-12-07T19:33:58.646+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5a20 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.646+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.646+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.646+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.646+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5340 prio 196 cost 0 latency 0.000061 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.646+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.646+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.646+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:3c88320e:::smithi13140796-1:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:1 client_op=osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[0~32768]}) 2019-12-07T19:33:58.646+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] repop_all_committed: repop tid 0 all committed 2019-12-07T19:33:58.646+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x5557e7967320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.646+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] commit: repgather(0x5557e7967320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.646+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] log_op_stats osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 inb 32768 outb 0 lat 0.021311 2019-12-07T19:33:58.646+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 24 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2 ==== 67508+0+0 (crc 0 0 0) 0x5557e763c800 con 0x5557e79ca800 2019-12-07T19:33:58.646+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2 2019-12-07T19:33:58.646+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000088 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2 pg pg[1.6s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=20'1 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=20'1 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:46, at_version=21'4, trim_to=0'0, roll_forward_to=20'1)) v2 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=20'1 lcod 21'2 active mbc={}] append_log log((0'0,21'3], crt=20'1) [21'4 (0'0) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:46 2019-12-07T19:33:58.643677+0000 0 ObjectCleanRegions clean_offsets: [262144~18446744073709289471], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=20'1 lcod 21'2 active mbc={}] add_log_entry 21'4 (0'0) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:46 2019-12-07T19:33:58.643677+0000 0 ObjectCleanRegions clean_offsets: [262144~18446744073709289471], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=20'1 lcod 21'2 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=20'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.646+0000 7f2b774e0700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9d5a300 seq 8 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) get_onode oid 1#1:6336c6f2:::smithi13140796-16:head# key 0x8180000000000000016336c6f221736d'ithi13140796-16!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.646+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1099 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# = 0 2019-12-07T19:33:58.646+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 2 keys 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 2 keys = 0 2019-12-07T19:33:58.646+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.646+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1245 < min_flush_size 524288 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.646+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 0x0~10000 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:6336c6f2:::smithi13140796-16:head# 0x0~10000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~10000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7d180 had 1 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 2 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9d5a300 8 blobs 2019-12-07T19:33:58.646+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2538 < min_flush_size 524288 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7c000 had 1 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 2 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4140 < min_flush_size 524288 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0f00 had 1 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5254 < min_flush_size 524288 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 2 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6355 < min_flush_size 524288 2019-12-07T19:33:58.646+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6894 < min_flush_size 524288 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 fbmap_alloc 0x5557e6282e00 allocate 0x10000/1000,10000,0 2019-12-07T19:33:58.646+0000 7f2b774e0700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xb0000~10000/1000,10000,0 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc1680 had 1 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xb0000~10000] 2019-12-07T19:33:58.646+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.646+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8480 < min_flush_size 524288 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668b60 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3420 sbid 0x0)) 2019-12-07T19:33:58.646+0000 7f2b774e0700 20 bluestore.blob(0x5557e7668b60) get_ref 0x0~2000 Blob(0x5557e7668b60 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3420 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc1900 had 1 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e7668b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7668b60 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3420 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8503 < min_flush_size 524288 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9cb3438 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8503 < min_flush_size 524288 2019-12-07T19:33:58.647+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.647+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2672d~2137 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76682a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7668af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2672d 0x2672d~2137 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76682a0 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668af0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.647+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x672d 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e76682a0) get_ref 0x0~2000 Blob(0x5557e76682a0 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668af0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x72d 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e76682a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e76682a0 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7668af0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e7668b08 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76689a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76689a0 blob([0xb4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36a10 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e76689a0) get_ref 0x0~2000 Blob(0x5557e76689a0 blob([0xb4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36a10 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e76689a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e76689a0 blob([0xb4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36a10 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9d36a28 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d36a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d36a80 blob([0xb6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36af0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36a80) get_ref 0x0~2000 Blob(0x5557e9d36a80 blob([0xb6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36af0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9d36a80 blob([0xb6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36af0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9d36b08 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d36b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d36b60 blob([0xb8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36bd0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36b60) get_ref 0x0~2000 Blob(0x5557e9d36b60 blob([0xb8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36bd0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9d36b60 blob([0xb8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36bd0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9d36be8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d36c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d36c40 blob([0xba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36cb0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x864 and padding block with 0x79c 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36c40) get_ref 0x0~2000 Blob(0x5557e9d36c40 blob([0xba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36cb0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9d36c40 blob([0xba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36cb0 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9d36cc8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d36d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d36d20 blob([0xbc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36d90 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36d20) get_ref 0x0~2000 Blob(0x5557e9d36d20 blob([0xbc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36d90 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9d36d20 blob([0xbc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36d90 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9d36da8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d36e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d36e00 blob([0xbe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36e70 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36e00) get_ref 0x0~2000 Blob(0x5557e9d36e00 blob([0xbe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36e70 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d36e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9d36e00 blob([0xbe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36e70 sbid 0x0)) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9d36e88 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x10000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e9d58608) dirty_range mark inline shard dirty 2019-12-07T19:33:58.647+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 0x0~10000 = 0 2019-12-07T19:33:58.647+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.647+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.647+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.647+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.647+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5a300 cost 103158 (9 ios * 4000 + 67158 bytes) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5a300 onodes 0x5557e9d58500 shared_blobs 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e9d58608) update 1#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e9d58608) update inline shard 155 bytes from 8 extents 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:6336c6f2:::smithi13140796-16:head# is 574 (413 bytes onode + 2 bytes spanning blobs + 159 bytes inline extents) 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5a300 allocated 0x[b0000~10000] released 0x[] 2019-12-07T19:33:58.647+0000 7f2b774e0700 10 freelist allocate 0xb0000~10000 2019-12-07T19:33:58.647+0000 7f2b774e0700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.647+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 prepare 2019-12-07T19:33:58.647+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9d5a300 2019-12-07T19:33:58.646+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] sending reply on osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 0x5557e76ca500 2019-12-07T19:33:58.647+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(1 smithi13140796-1 [append 0~32768,stat] v21'1 uv1 ondisk = 0) v8 -- 0x5557e76ca500 con 0x5557e6f87800 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.647+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:13 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] removing repgather(0x5557e7967320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] q front is repgather(0x5557e7967320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.647+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] remove_repop repgather(0x5557e7967320 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] requeue_op 0x5557e7695760 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] requeue_op 0x5557e76954a0 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_finish_rmw: Op(1:3c88320e:::smithi13140796-1:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:1 client_op=osd_op(client.4540.0:1 1.cs0 1:3c88320e:::smithi13140796-1:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[0~32768]}) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_finish_rmw: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~32768:2) Extents(1:3c88320e:::smithi13140796-1:head)[ Extent(0~32768:1) Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5340 finish 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e76954a0 prio 63 cost 62 latency 0.022064 osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.647+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 44 ==== MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2228+0+0 (unknown 0 0 0) 0x5557e763e000 con 0x5557e79cac00 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x5557e760c900 2019-12-07T19:33:58.647+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695ce0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.647+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x5557e760c900 1:3c88320e:::smithi13140796-1:head rwstate(none n=0 w=0) oi: 1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967680 snapset: 0=[]:{} 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e766a000 2019-12-07T19:33:58.647+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:3c88320e:::smithi13140796-1:head(21'1 client.4540.0:1 dirty|data_digest s 32768 uv 1 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op obc obc(1:3c88320e:::smithi13140796-1:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 0x5557e9b14300 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7695ce0 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] ov 21'1 av 21'4 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op setxattr _header (55) in=62b 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:14 2019-12-07T19:33:58.627337+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:3c88320e:::smithi13140796-1:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:3c88320e:::smithi13140796-1:head done, snapset=0=[]:{} 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx 1:3c88320e:::smithi13140796-1:head 0x5557e9b14300 op modify 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:14 2019-12-07T19:33:58.627337+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.625650+0000 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.647+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx object 1:3c88320e:::smithi13140796-1:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x5557e76caf00 result 0 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e9b10c00 seq 7 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op order client.4540 tid 2 last was 1 2019-12-07T19:33:58.647+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.es5_head 5#1:733d7474:::smithi13140796-5:head# 3 keys 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.es5_head 5#1:733d7474:::smithi13140796-5:head# 3 keys = 0 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop rep_tid 3 on osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.647+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.es5_head 5#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop: repgather(0x5557e7967c20 0'0 rep_tid=3 committed?=0 r=0) 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.es5_head 5#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.647+0000 7f2b744da700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] issue_repop rep_tid 3 o 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.647+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:3c88320e:::smithi13140796-1:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:2 client_op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10c00 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10c00 onodes 0x5557e9bc0280 shared_blobs 2019-12-07T19:33:58.647+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0388) update 5#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: found on disk, size 8192 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:733d7474:::smithi13140796-5:head# is 546 (483 bytes onode + 2 bytes spanning blobs + 61 bytes inline extents) 2019-12-07T19:33:58.647+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.647+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key = 0 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 prepare 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10c00 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:3c88320e:::smithi13140796-1:head projected size 32768 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 io_done 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] start_rmw: Op(1:3c88320e:::smithi13140796-1:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:2 client_op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:3c88320e:::smithi13140796-1:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:2 client_op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:33:58.647+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7695ce0 finish 2019-12-07T19:33:58.647+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:3c88320e:::smithi13140796-1:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:2 client_op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~32768:2) Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] generate_transactions: 1:3c88320e:::smithi13140796-1:head, current size is 32768 buffers are {} 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): new_size start 32768 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:3c88320e:::smithi13140796-1:head resetting hinfo to logical size 32768 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~32768:2) Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:3c88320e:::smithi13140796-1:head={}} 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:3c88320e:::smithi13140796-1:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:2 client_op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:3c88320e:::smithi13140796-1:head=[]} 2019-12-07T19:33:58.647+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:3c88320e:::smithi13140796-1:head,{} 2019-12-07T19:33:58.647+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:2, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x5557e9c78000 con 0x5557e79cac00 2019-12-07T19:33:58.647+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:2, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x5557e9c78800 con 0x5557e7519800 2019-12-07T19:33:58.647+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:2, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x5557e9c79000 con 0x5557e79ca800 2019-12-07T19:33:58.647+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:2, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x5557e9c79800 con 0x5557e79cbc00 2019-12-07T19:33:58.647+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:2, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 -- 0x5557e9c7a000 con 0x5557e79ca000 2019-12-07T19:33:58.647+0000 7f2b744da700 20 earliest_dup_version = 0 2019-12-07T19:33:58.647+0000 7f2b744da700 20 trim 21'4 (21'1) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:2 2019-12-07T19:33:58.625650+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.647+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log log((0'0,21'3], crt=0'0) [21'4 (21'1) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:2 2019-12-07T19:33:58.625650+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 lua=21'3 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'4 (21'1) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:2 2019-12-07T19:33:58.625650+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 lua=21'3 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] rollforward: entry=21'1 (0'0) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:1 2019-12-07T19:33:58.625551+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 lua=21'3 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 4 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 lua=21'3 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.648+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.648+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9b10300 seq 9 2019-12-07T19:33:58.648+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 3 keys 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 3 keys = 0 2019-12-07T19:33:58.648+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.648+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10300 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10300 onodes 0x5557e76caa00 shared_blobs 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore.extentmap(0x5557e76cab08) update 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:3c88320e:::smithi13140796-1:head# is 510 (482 bytes onode + 2 bytes spanning blobs + 26 bytes inline extents) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 prepare 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10300 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 io_done 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 lua=21'3 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'4 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x5557e7967c20 21'4 rep_tid=3 committed?=0 r=0) 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e76954a0 finish 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5a20 prio 196 cost 0 latency 0.001670 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:4 client_op=osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[0~32768]}) 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'1 crt=21'1 lcod 0'0 mlcod 0'0 active+clean] repop_all_committed: repop tid 1 all committed 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] eval_repop repgather(0x5557e628b200 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] commit: repgather(0x5557e628b200 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.648+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] log_op_stats osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 inb 32768 outb 0 lat 0.022744 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] sending reply on osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 0x5557e76cac80 2019-12-07T19:33:58.648+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(4 smithi13140796-2 [append 0~32768,stat] v21'2 uv2 ondisk = 0) v8 -- 0x5557e76cac80 con 0x5557e6f87800 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.648+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] publish_stats_to_osd 21:14 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] removing repgather(0x5557e628b200 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] q front is repgather(0x5557e628b200 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.648+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] remove_repop repgather(0x5557e628b200 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] requeue_op 0x5557e7695e40 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] requeue_op 0x5557e7695b80 2019-12-07T19:33:58.648+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x28864 2019-12-07T19:33:58.648+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.648+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.648+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.648+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:4 client_op=osd_op(client.4540.0:4 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~32768 in=32768b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[0~32768]}) 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(0~32768:2) Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5a20 finish 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8) prio 63 cost 62 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695b80 prio 63 cost 62 latency 0.022842 osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 pg pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.648+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'2 client.4540.0:4 dirty|data_digest s 32768 uv 2 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 0x5557e70b6a00 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] ov 21'2 av 21'5 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] do_osd_op setxattr _header (55) in=62b 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=0=[]:{} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x5557e70b6a00 op modify 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] set mtime to 2019-12-07T19:33:58.625925+0000 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] final snapset 0=[]:{} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] execute_ctx alloc reply 0x5557e9c7c500 result 0 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] calc_trim_to_aggressive limit = 21'1 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] op order client.4540 tid 5 last was 4 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] new_repop rep_tid 4 on osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] new_repop: repgather(0x5557e7967320 0'0 rep_tid=4 committed?=0 r=0) 2019-12-07T19:33:58.648+0000 7f2b744da700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] issue_repop rep_tid 4 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:5 client_op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: found on disk, size 8192 2019-12-07T19:33:58.648+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 32768 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:5 client_op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:5 client_op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:5 client_op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: ExtentCache( Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 32768 buffers are {} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] operator(): new_size start 32768 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 32768 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: ExtentCache( Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:5 client_op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{} 2019-12-07T19:33:58.648+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:5, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x5557e9c7a800 con 0x5557e79cac00 2019-12-07T19:33:58.648+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:5, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x5557e9c7b000 con 0x5557e7519800 2019-12-07T19:33:58.648+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:5, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x5557e9c7b800 con 0x5557e79ca800 2019-12-07T19:33:58.648+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:5, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x5557e9d80000 con 0x5557e79cbc00 2019-12-07T19:33:58.648+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:5, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x5557e9d80800 con 0x5557e79ca000 2019-12-07T19:33:58.648+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 25 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 2228+0+0 (crc 0 0 0) 0x5557e9c79000 con 0x5557e79ca800 2019-12-07T19:33:58.648+0000 7f2b744da700 20 earliest_dup_version = 0 2019-12-07T19:33:58.648+0000 7f2b744da700 20 trim 21'5 (21'2) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:5 2019-12-07T19:33:58.625925+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] append_log log((0'0,21'4], crt=21'1) [21'5 (21'2) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:5 2019-12-07T19:33:58.625925+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.648+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84000 prio 127 cost 0 latency 0.000035 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.648+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'1 lcod 21'1 mlcod 21'1 active+clean] add_log_entry 21'5 (21'2) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:5 2019-12-07T19:33:58.625925+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] rollforward: entry=21'2 (0'0) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:4 2019-12-07T19:33:58.625910+0000 0 ObjectCleanRegions clean_offsets: [32768~18446744073709518847], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.648+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log approx pg log length = 5 2019-12-07T19:33:58.648+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.648+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.648+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9d5a900 seq 10 2019-12-07T19:33:58.648+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 3 keys 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 3 keys = 0 2019-12-07T19:33:58.648+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.648+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5a900 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5a900 onodes 0x5557e76cb400 shared_blobs 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore.extentmap(0x5557e76cb508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 510 (482 bytes onode + 2 bytes spanning blobs + 26 bytes inline extents) 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5a900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a900 prepare 2019-12-07T19:33:58.648+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5a900 2019-12-07T19:33:58.648+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a900 io_done 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_applied version 21'5 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] eval_repop repgather(0x5557e7967320 21'5 rep_tid=4 committed?=0 r=0) 2019-12-07T19:33:58.648+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695b80 finish 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695e40 prio 63 cost 0 latency 0.023395 osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 pg pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.648+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.648+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.649+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695e40 finish 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695760 prio 63 cost 0 latency 0.023652 osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 pg pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x5557e760c900 2019-12-07T19:33:58.649+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 14 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9b37400 con 0x5557e7519800 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x5557e760c900 1:3c88320e:::smithi13140796-1:head rwstate(write n=1 w=0) oi: 1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967680 snapset: 0=[]:{} 2019-12-07T19:33:58.649+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695340 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.649+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695760 finish 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 26 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c77000 con 0x5557e79ca800 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.649+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84160 prio 196 cost 0 latency 0.000009 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.649+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695340 prio 196 cost 0 latency 0.000064 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.649+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 27 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c76800 con 0x5557e79ca800 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.649+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d842c0 prio 196 cost 0 latency 0.000005 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.649+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695340 finish 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84160 prio 196 cost 0 latency 0.000054 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:3f7a742d:::smithi13140796-9:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:25 client_op=osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~147456]}) 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] repop_all_committed: repop tid 2 all committed 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] eval_repop repgather(0x5557e7967b00 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] commit: repgather(0x5557e7967b00 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.649+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] log_op_stats osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 147456 outb 0 lat 0.017019 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] sending reply on osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x5557e76cb180 2019-12-07T19:33:58.649+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(25 smithi13140796-9 [append 0~147456,stat] v21'3 uv3 ondisk = 0) v8 -- 0x5557e76cb180 con 0x5557e6f87800 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.649+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] publish_stats_to_osd 21:15 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] removing repgather(0x5557e7967b00 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] q front is repgather(0x5557e7967b00 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.649+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] remove_repop repgather(0x5557e7967b00 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] requeue_op 0x5557e75c49a0 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] requeue_op 0x5557e75c46e0 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 62 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_finish_rmw: Op(1:3f7a742d:::smithi13140796-9:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:25 client_op=osd_op(client.4540.0:25 1.cs0 1:3f7a742d:::smithi13140796-9:head [append 0~147456 in=147456b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~147456]}) 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_finish_rmw: ExtentCache( Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~147456:3) ) 2019-12-07T19:33:58.649+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84160 finish 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 62 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c46e0 prio 63 cost 62 latency 0.017110 osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] _handle_message: osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_op: op osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_op osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_object_context: found obc in cache: 0x5557e9b21680 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(none n=0 w=0) oi: 1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.649+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(21'3 client.4540.0:25 dirty|data_digest s 147456 uv 3 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_op obc obc(1:3f7a742d:::smithi13140796-9:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] execute_ctx 0x5557e70b7800 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] execute_ctx 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] ov 21'3 av 21'6 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_osd_op 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_osd_op setxattr _header (55) in=62b 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] make_writeable 1:3f7a742d:::smithi13140796-9:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] make_writeable 1:3f7a742d:::smithi13140796-9:head done, snapset=0=[]:{} 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] finish_ctx 1:3f7a742d:::smithi13140796-9:head 0x5557e70b7800 op modify 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] set mtime to 2019-12-07T19:33:58.630747+0000 2019-12-07T19:33:58.649+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 28 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b37000 con 0x5557e79ca800 2019-12-07T19:33:58.649+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 196 cost 0 latency 0.000006 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.649+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] final snapset 0=[]:{} in 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] finish_ctx object 1:3f7a742d:::smithi13140796-9:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] execute_ctx alloc reply 0x5557e76ca780 result 0 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] calc_trim_to_aggressive limit = 21'2 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] op order client.4540 tid 26 last was 25 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] new_repop rep_tid 5 on osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] new_repop: repgather(0x5557e628b200 0'0 rep_tid=5 committed?=0 r=0) 2019-12-07T19:33:58.649+0000 7f2b784e2700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] issue_repop rep_tid 5 o 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.649+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 15 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e7a9b800 con 0x5557e7519800 2019-12-07T19:33:58.649+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695a20 prio 196 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.649+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] submit_transaction: op Op(1:3f7a742d:::smithi13140796-9:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:26 client_op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_hash_info: Getting attr on 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_hash_info: not in cache 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_hash_info: found on disk, size 36864 2019-12-07T19:33:58.649+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key = 0 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): 1:3f7a742d:::smithi13140796-9:head projected size 147456 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] start_rmw: Op(1:3f7a742d:::smithi13140796-9:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:26 client_op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[]}) 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_state_to_reads: Op(1:3f7a742d:::smithi13140796-9:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:26 client_op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[]}) 2019-12-07T19:33:58.649+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: starting commit on Op(1:3f7a742d:::smithi13140796-9:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:26 client_op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[]}) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] generate_transactions: 1:3f7a742d:::smithi13140796-9:head, current size is 147456 buffers are {} 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 6 cleaned 0 in 0.005330099s (0.002385354s flush + 0.002944745s kv commit) 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1567 < min_flush_size 524288 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0500 had 1 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 2 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): new_size start 147456 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692900,0x5557e7692f00,0x5557e7693200,0x5557e9cc4000,0x5557e7665500,0x5557e7693800> 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 kv_submitted 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2681 < min_flush_size 524288 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692900 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 finishing 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692900 onodes 0x5557e9c7d180 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c6b78 in 0x5557e6269960) _finish_write discard buffer(0x5557e79ab170 space 0x5557e75c6b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 2 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4264 < min_flush_size 524288 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7620fd8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aac60 space 0x5557e7620fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): 1:3f7a742d:::smithi13140796-9:head resetting hinfo to logical size 147456 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b572e8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aa360 space 0x5557e9b572e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be3208 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aa990 space 0x5557e9be3208 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be32e8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79ab950 space 0x5557e9be32e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be33c8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b59b90 space 0x5557e9be33c8 0x0~1000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0280 had 1 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 done 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693200 kv_submitted 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5811 < min_flush_size 524288 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692900 [] 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 kv_submitted 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692f00 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76caa00 had 1 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: written: {1:3f7a742d:::smithi13140796-9:head={}} 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 finishing 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 2 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692f00 onodes 0x5557e9b7c000 2019-12-07T19:33:58.649+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=6, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9b36600 con 0x5557e79cac00 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621898 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b59560 space 0x5557e7621898 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7669978 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b595f0 space 0x5557e7669978 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 7358 < min_flush_size 524288 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2088 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b59680 space 0x5557e9cb2088 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2168 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9cb2168 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2248 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9cb2248 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2328 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b58b40 space 0x5557e9cb2328 0x0~2000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: op: Op(1:3f7a742d:::smithi13140796-9:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:26 client_op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[]}) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2408 in 0x5557e6269a40) _finish_write discard buffer(0x5557e9b58bd0 space 0x5557e9cb2408 0x0~1000 writing nocache) 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb400 had 1 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692f00 done 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693800 kv_submitted 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692f00 [] 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 7381 < min_flush_size 524288 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 7381 < min_flush_size 524288 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: written_set: {1:3f7a742d:::smithi13140796-9:head=[]} 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.649+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: 1:3f7a742d:::smithi13140796-9:head,{} 2019-12-07T19:33:58.649+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 kv_submitted 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x28864~1cd5 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.649+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693200 2019-12-07T19:33:58.649+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x28864 0x28864~1cd5 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x8864 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x864 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.649+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x539 and padding block with 0xac7 2019-12-07T19:33:58.649+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.649+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=7, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e9c76600 con 0x5557e79cac00 2019-12-07T19:33:58.649+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.649+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.649+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:26, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x5557e79e3800 con 0x5557e79cac00 2019-12-07T19:33:58.649+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:26, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x5557e79e4000 con 0x5557e7519800 2019-12-07T19:33:58.649+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 45 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e76b0000 con 0x5557e79cac00 2019-12-07T19:33:58.650+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:26, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x5557e763d800 con 0x5557e79ca800 2019-12-07T19:33:58.650+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84580 prio 196 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.650+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:26, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x5557e763d000 con 0x5557e79cbc00 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 46 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9c76000 con 0x5557e79cac00 2019-12-07T19:33:58.650+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:26, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x5557e763f000 con 0x5557e79ca000 2019-12-07T19:33:58.650+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d846e0 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 earliest_dup_version = 0 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 trim 21'6 (21'3) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:26 2019-12-07T19:33:58.630747+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 finishing 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693200 onodes 0x5557e9bc0f00 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693200 done 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693200 [] 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] append_log log((0'0,21'5], crt=21'2) [21'6 (21'3) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:26 2019-12-07T19:33:58.630747+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] add_log_entry 21'6 (21'3) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:26 2019-12-07T19:33:58.630747+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 kv_submitted 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4000 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] rollforward: entry=21'3 (0'0) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:25 2019-12-07T19:33:58.630729+0000 0 ObjectCleanRegions clean_offsets: [147456~18446744073709404159], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 finishing 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4000 onodes 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4000 done 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665500 kv_submitted 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4000 [] 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] append_log approx pg log length = 6 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 kv_submitted 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665500 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665500 finishing 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665500 onodes 0x5557e9bc1680 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665500 done 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665500 [] 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.650+0000 7f2b784e2700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 kv_submitted 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693800 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 finishing 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693800 onodes 0x5557e9bc1900 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693800 done 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693800 [] 2019-12-07T19:33:58.650+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.650+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.650+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7a92f00 seq 11 2019-12-07T19:33:58.650+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 3 keys 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 3 keys = 0 2019-12-07T19:33:58.650+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.650+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92f00 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92f00 onodes 0x5557e9c7c280 shared_blobs 2019-12-07T19:33:58.650+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e9c7c388) update 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:3f7a742d:::smithi13140796-9:head# is 584 (483 bytes onode + 2 bytes spanning blobs + 99 bytes inline extents) 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 prepare 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92f00 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 io_done 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] op_applied version 21'6 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] eval_repop repgather(0x5557e628b200 21'6 rep_tid=5 committed?=0 r=0) 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c46e0 finish 2019-12-07T19:33:58.650+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=9, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e9b37600 con 0x5557e79cac00 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=12, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e76b1a00 con 0x5557e79cac00 2019-12-07T19:33:58.650+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=10, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e76b1c00 con 0x5557e79cac00 2019-12-07T19:33:58.650+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d842c0 prio 196 cost 0 latency 0.000981 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d842c0 finish 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=8, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e7a9b600 con 0x5557e79cac00 2019-12-07T19:33:58.650+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 196 cost 0 latency 0.000705 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84580 prio 196 cost 0 latency 0.000378 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84580 finish 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695a20 prio 196 cost 0 latency 0.000712 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695a20 finish 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d846e0 prio 196 cost 0 latency 0.000394 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d846e0 finish 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.650+0000 7f2b77ce1700 10 osd.6 21 dequeue_op 0x5557e75c4580 finish 2019-12-07T19:33:58.650+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 aio_wait 2019-12-07T19:33:58.650+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692000 2019-12-07T19:33:58.650+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 io_done 2019-12-07T19:33:58.650+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2a539 2019-12-07T19:33:58.650+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.650+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.650+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.650+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 29 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9d88000 con 0x5557e79ca800 2019-12-07T19:33:58.650+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84840 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 msgr2=0x5557e7529180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 93 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 msgr2=0x5557e7529180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 unknown :-1 s=READY pgs=17 cs=1 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 unknown :-1 s=READY pgs=17 cs=1 l=0 rx=0 tx=0)._fault initiating reconnect 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c49a0 prio 63 cost 0 latency 0.018371 osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] do_op: op osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] do_op osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] get_object_context: found obc in cache: 0x5557e9b21680 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 17 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e76b1c00 con 0x5557e79cbc00 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(write n=1 w=0) oi: 1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.650+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d849a0 prio 196 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 18 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9d88e00 con 0x5557e79cbc00 2019-12-07T19:33:58.650+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.650+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84b00 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c49a0 finish 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84840 prio 196 cost 0 latency 0.000250 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e748e400 0x5557e75b1600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84840 finish 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d849a0 prio 196 cost 0 latency 0.000132 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.650+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d849a0 finish 2019-12-07T19:33:58.650+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.650+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.650+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e748e400 0x5557e75b1600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84b00 prio 196 cost 0 latency 0.000120 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.650+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.651+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84b00 finish 2019-12-07T19:33:58.651+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.651+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.109602748s delta_ub 0.003458990s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9d89800 con 0x5557e79f4800 2019-12-07T19:33:58.651+0000 7f2b967b4700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002783302s,0.003458990s]) 2019-12-07T19:33:58.651+0000 7f2b957b2700 10 osd.6 21 ms_handle_authentication new session 0x5557e7559860 con 0x5557e748e400 entity osd.1 addr 2019-12-07T19:33:58.651+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.106819446s delta_ub 18446744073.706768314s) v5 -- 0x5557e9d89a00 con 0x5557e79f4800 2019-12-07T19:33:58.651+0000 7f2b957b2700 10 osd.6 21 ms_handle_authentication session 0x5557e7559860 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 unknown :-1 s=BANNER_CONNECTING pgs=17 cs=2 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e748e400 0x5557e75b1600 crc :-1 s=SESSION_ACCEPTING pgs=19 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect race detected, this connection loses to existing=0x5557e79cac00 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e748e400 msgr2=0x5557e75b1600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 97 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e748e400 msgr2=0x5557e75b1600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.651+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 16 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9d88200 con 0x5557e7519800 2019-12-07T19:33:58.651+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695a20 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.651+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e748e400 0x5557e75b1600 crc :-1 s=SESSION_ACCEPTING pgs=19 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e748e400 0x5557e75b1600 crc :-1 s=SESSION_ACCEPTING pgs=19 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.651+0000 7f2b834f8700 2 osd.6 21 ms_handle_reset con 0x5557e748e400 session 0x5557e7559860 2019-12-07T19:33:58.651+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.651+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 crc :-1 s=SESSION_RECONNECTING pgs=17 cs=4 l=0 rx=0 tx=0).handle_session_retry received session retry connect_seq=3, inc to cs=4 2019-12-07T19:33:58.651+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695a20 prio 196 cost 0 latency 0.000063 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.651+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.651+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695a20 finish 2019-12-07T19:33:58.651+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.109602748s delta_ub 0.003458990s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e76b1a00 con 0x5557e79f4c00 2019-12-07T19:33:58.651+0000 7f2b957b2700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002394340s,0.003458990s]) 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.107208408s delta_ub 18446744073.707157276s) v5 -- 0x5557e9d88c00 con 0x5557e79f4c00 2019-12-07T19:33:58.651+0000 7f2b834f8700 10 osd.6 21 ms_handle_connect con 0x5557e79cac00 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x5557e79cac00 0x5557e7529180 crc :-1 s=READY pgs=17 cs=4 l=0 rx=0 tx=0).ready entity=osd.1 client_cookie=949c5c84788cefc8 server_cookie=3b65d3eec6e35050 in_seq=46 out_seq=35 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 47 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e7a9b600 con 0x5557e79cac00 2019-12-07T19:33:58.651+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84c60 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.651+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.651+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 48 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763f000 con 0x5557e79cac00 2019-12-07T19:33:58.651+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84dc0 prio 127 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.651+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.651+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.651+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84c60 prio 196 cost 0 latency 0.000039 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 49 ==== MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e76cb180 con 0x5557e79cac00 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.651+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000011 epoch 21 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.651+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.651+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84c60 finish 2019-12-07T19:33:58.651+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 prio 127 cost 0 latency 0.000040 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.651+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e9cc4300 seq 8 2019-12-07T19:33:58.651+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4300 onodes shared_blobs 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 prepare 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4300 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 io_done 2019-12-07T19:33:58.651+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 finish 2019-12-07T19:33:58.651+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.651+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000134 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.651+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.651+0000 7f2b76cdf700 25 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.651+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 0x0~1000 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0xa000 (40960) 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c70a0 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76204d0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e75c70a0 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76204d0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.651+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 5 cleaned 0 in 0.002436603s (0.000000116s flush + 0.002436487s kv commit) 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5a000,0x5557e9b10900,0x5557e9b10c00,0x5557e9b10300,0x5557e9d5a900> 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a000 kv_submitted 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5a000 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a000 finishing 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5a000 onodes 0x5557e9bc0500 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a000 done 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a300 aio_wait 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5a000 [] 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 kv_submitted 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10900 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 finishing 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10900 onodes 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10900 done 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10900 [] 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10c00 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 finishing 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10c00 onodes 0x5557e9bc0280 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 done 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10c00 [] 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 kv_submitted 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10300 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 finishing 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10300 onodes 0x5557e76caa00 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10300 done 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a900 kv_submitted 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10300 [] 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a900 kv_submitted 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5a900 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a900 finishing 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5a900 onodes 0x5557e76cb400 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a900 done 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92f00 kv_queued 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5a900 [] 2019-12-07T19:33:58.652+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.652+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.652+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.652+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.652+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.652+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1621 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7c280 had 1 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.652+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3683 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7ca00 had 1 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.652+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4797 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.652+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4820 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4820 < min_flush_size 524288 2019-12-07T19:33:58.652+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2a539~12d4 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e75c70a0 blob([0x4e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76204d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.652+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2a539 0x2a539~12d4 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e76204e8 in 0x5557e6269880) _discard 0x0~1000 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xa539 2019-12-07T19:33:58.652+0000 7f2b76cdf700 20 _add level 1 near 0 on buffer(0x5557e9b585a0 space 0x5557e76204e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x539 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.652+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x80d and padding block with 0x7f3 2019-12-07T19:33:58.652+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 0x0~1000 = 4096 2019-12-07T19:33:58.652+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.652+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:33:58.652+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:33:58.652+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.652+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 40960 2019-12-07T19:33:58.652+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# hinfo_key 2019-12-07T19:33:58.652+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ecc72bdb:::smithi13140796-10:head# hinfo_key = 0 2019-12-07T19:33:58.652+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.652+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=17, attrs_read=0)) v2 -- 0x5557e9b7d180 con 0x5557e79cac00 2019-12-07T19:33:58.652+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e75c51e0 finish 2019-12-07T19:33:58.652+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.652+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.652+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.652+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 8 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e76b1a00 con 0x5557e79ca000 2019-12-07T19:33:58.652+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84580 prio 196 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.652+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.652+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.652+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 9 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9d88c00 con 0x5557e79ca000 2019-12-07T19:33:58.652+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.652+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5a20 prio 196 cost 0 latency 0.000009 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.652+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.652+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.652+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.652+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84580 prio 196 cost 0 latency 0.000054 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:3c88320e:::smithi13140796-1:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:2 client_op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] repop_all_committed: repop tid 3 all committed 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] eval_repop repgather(0x5557e7967c20 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2b80d 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] commit: repgather(0x5557e7967c20 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.653+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.653+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.653+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.653+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] log_op_stats osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 inb 0 outb 0 lat 0.027638 2019-12-07T19:33:58.653+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.653+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=20'1 lcod 21'2 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] sending reply on osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 0x5557e76caf00 2019-12-07T19:33:58.653+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e7694b00 finish 2019-12-07T19:33:58.653+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(2 smithi13140796-1 [setxattr (55)] v21'4 uv4 ondisk = 0) v8 -- 0x5557e76caf00 con 0x5557e6f87800 2019-12-07T19:33:58.653+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=3, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e7683c00 con 0x5557e79ca800 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.653+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=14, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e9b36e00 con 0x5557e79cac00 2019-12-07T19:33:58.653+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] publish_stats_to_osd 21:16 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 3 cleaned 0 in 0.000822172s (0.000463875s flush + 0.000358297s kv commit) 2019-12-07T19:33:58.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] removing repgather(0x5557e7967c20 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92f00,0x5557e7692000,0x5557e9cc4300> 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 kv_submitted 2019-12-07T19:33:58.653+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=15, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e7683e00 con 0x5557e79cac00 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92f00 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] q front is repgather(0x5557e7967c20 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 finishing 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92f00 onodes 0x5557e9c7c280 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92f00 done 2019-12-07T19:33:58.653+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.653+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d84000 prio 127 cost 0 latency 0.004377 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.6s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=20'1 lcod 21'3 active mbc={}] 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92f00 [] 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 kv_submitted 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692000 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] remove_repop repgather(0x5557e7967c20 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 finishing 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692000 onodes 0x5557e9b7ca00 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=20'1 lcod 21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:23, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb25c8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b59320 space 0x5557e9cb25c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.653+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2948 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b59b00 space 0x5557e9cb2948 0x0~2000 writing nocache) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] requeue_op 0x5557e7695760 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2a28 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b59e60 space 0x5557e9cb2a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2b08 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b59ef0 space 0x5557e9cb2b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.653+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e765e600 con 0x5557e7519800 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2be8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd6000 space 0x5557e9cb2be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb2cc8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd6090 space 0x5557e9cb2cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692000 done 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=20'1 lcod 21'3 active mbc={}] append_log log((0'0,21'4], crt=20'1) [21'5 (21'2) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:23 2019-12-07T19:33:58.629675+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692000 [] 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 kv_submitted 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4300 2019-12-07T19:33:58.653+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 finishing 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4300 onodes 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4300 done 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4300 [] 2019-12-07T19:33:58.653+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.653+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=20'1 lcod 21'3 active mbc={}] add_log_entry 21'5 (21'2) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:23 2019-12-07T19:33:58.629675+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] try_finish_rmw: Op(1:3c88320e:::smithi13140796-1:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:2 client_op=osd_op(client.4540.0:2 1.cs0 1:3c88320e:::smithi13140796-1:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'1 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3c88320e:::smithi13140796-1:head=[]}) 2019-12-07T19:33:58.653+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=16, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e758cc00 con 0x5557e79cac00 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.653+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] rollforward: entry=21'2 (0'0) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:22 2019-12-07T19:33:58.629655+0000 0 ObjectCleanRegions clean_offsets: [131072~18446744073709420543], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84580 finish 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.653+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9b10f00 seq 9 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695760 prio 63 cost 0 latency 0.027773 osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] 2019-12-07T19:33:58.653+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 3 keys 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 3 keys = 0 2019-12-07T19:33:58.653+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] _handle_message: osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.653+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_op: op osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.653+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e75c5ce0 finish 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.653+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10f00 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10f00 onodes 0x5557e9bc0780 shared_blobs 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_op osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0888) update 1#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:6c205177:::smithi13140796-8:head# is 568 (483 bytes onode + 2 bytes spanning blobs + 83 bytes inline extents) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_object_context: found obc in cache: 0x5557e760c900 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 prepare 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 blocked by 0x5557e9d5a300 aio_wait 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_object_context: 0x5557e760c900 1:3c88320e:::smithi13140796-1:head rwstate(none n=0 w=0) oi: 1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967680 snapset: 0=[]:{} 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.653+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d84000 finish 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] find_object_context 1:3c88320e:::smithi13140796-1:head @head oi=1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.653+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_op oi 1:3c88320e:::smithi13140796-1:head(21'4 client.4540.0:2 dirty|data_digest s 32768 uv 4 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_op obc obc(1:3c88320e:::smithi13140796-1:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] execute_ctx 0x5557e9ceea00 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] execute_ctx 1:3c88320e:::smithi13140796-1:head [read 0~1] ov 21'4 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_osd_op 1:3c88320e:::smithi13140796-1:head [read 0~1] 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_read 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] async_read noted for 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 6(0) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] start_read_op: starting ReadOp(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.653+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 19 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e758cc00 con 0x5557e79cbc00 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_read_op: starting read ReadOp(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.653+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5600 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.653+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.653+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9c7c780 con 0x5557e79cac00 2019-12-07T19:33:58.653+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9c7ca00 con 0x5557e79cbc00 2019-12-07T19:33:58.653+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9d58c80 con 0x5557e79ca000 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 send_incremental_map 0 -> 21 to 0x5557e62f5400 v2:172.21.15.109:6818/36937 2019-12-07T19:33:58.653+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3a80e826:::osdmap.21:0# 0x0~0 2019-12-07T19:33:58.653+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:33:58.653+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b563f0 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9b56460 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.653+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b564d0 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x5557e9b56540 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:33:58.653+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3a80e826:::osdmap.21:0# 0x0~140c = 5132 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 add_map_bl 21 5132 bytes 2019-12-07T19:33:58.653+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- osd_map(21..21 src has 1..21) v4 -- 0x5557e70c3b00 con 0x5557e62f5400 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 maybe_share_map con 0x5557e62f5400 v2:172.21.15.109:6818/36937 map epoch 0 -> 21 (shared) 2019-12-07T19:33:58.653+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9d58f00 con 0x5557e62f5400 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_read_op: started ReadOp(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,6(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=1(3),4(2),5(1),6(0)) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695760 finish 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5a20 prio 196 cost 0 latency 0.000672 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:5 client_op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] repop_all_committed: repop tid 4 all committed 2019-12-07T19:33:58.653+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x5557e9d58f00 con 0x5557e62f5400 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] eval_repop repgather(0x5557e7967320 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e7694160 prio 127 cost 0 latency 0.000094 epoch 21 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] commit: repgather(0x5557e7967320 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.653+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] log_op_stats osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 inb 0 outb 0 lat 0.028140 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] sending reply on osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) v8 0x5557e9c7c500 2019-12-07T19:33:58.653+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(5 smithi13140796-2 [setxattr (55)] v21'5 uv5 ondisk = 0) v8 -- 0x5557e9c7c500 con 0x5557e6f87800 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.653+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] publish_stats_to_osd 21:17 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] removing repgather(0x5557e7967320 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] q front is repgather(0x5557e7967320 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] remove_repop repgather(0x5557e7967320 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] requeue_op 0x5557e7695e40 2019-12-07T19:33:58.653+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 10 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e7683e00 con 0x5557e79ca000 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.653+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c58c0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.653+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:5 client_op=osd_op(client.4540.0:5 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e20) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5a20 finish 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695e40 prio 63 cost 0 latency 0.028279 osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.653+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 17 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:34, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x5557e79e4000 con 0x5557e7519800 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.653+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e76958c0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:34, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.653+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:34, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op: op osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.653+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.653+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:34, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.653+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.653+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:34, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.653+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e76958c0 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:34, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.653+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.653+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:34, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 0x5557e70b6a00 2019-12-07T19:33:58.653+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~1] ov 21'5 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~1] 2019-12-07T19:33:58.653+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:34 2019-12-07T19:33:58.636620+0000 0 ObjectCleanRegions clean_offsets: [196608~18446744073709355007], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.653+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_read 2019-12-07T19:33:58.653+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] async_read noted for 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.653+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.653+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:33:58.653+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.653+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:33:58.654+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9b11500 seq 6 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 6(0) 2019-12-07T19:33:58.654+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.654+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.654+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11500 onodes shared_blobs 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] start_read_op: starting ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.654+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 prepare 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11500 2019-12-07T19:33:58.654+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 io_done 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_read_op: starting read ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.654+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e76958c0 finish 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.654+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9c7cf00 con 0x5557e79cac00 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 11 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 4260+0+0 (unknown 0 0 0) 0x5557e9c7c500 con 0x5557e79ca000 2019-12-07T19:33:58.654+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9c7c000 con 0x5557e79cbc00 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2b80d~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e76cac80 con 0x5557e79ca000 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2b80d 0x2b80d~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xb80d 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x80d 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 20 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x5557e9d58c80 con 0x5557e79cbc00 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.654+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9b7d900 con 0x5557e62f5400 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xc7e and padding block with 0x382 2019-12-07T19:33:58.654+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4c60 prio 127 cost 0 latency 0.000008 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x5557e9b7d900 con 0x5557e62f5400 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_read_op: started ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),4(2),5(1),6(0)) 2019-12-07T19:33:58.654+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695e40 finish 2019-12-07T19:33:58.654+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5600 prio 196 cost 0 latency 0.000725 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 50 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x5557e9d59180 con 0x5557e79cac00 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5600 finish 2019-12-07T19:33:58.654+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c54a0 prio 127 cost 0 latency 0.000011 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c58c0 prio 196 cost 0 latency 0.000445 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c58c0 finish 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.654+0000 7f2b957b2700 0 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 msgr2=0x5557e7528100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7694160 prio 127 cost 0 latency 0.000663 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 msgr2=0x5557e7528100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 92 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=0,4096,0}, subchunks={1:3c88320e:::smithi13140796-1:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 msgr2=0x5557e7528100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.654+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=READY pgs=5 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.654+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x0~1000 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x2000 (8192) 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b578f0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57880 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=READY pgs=5 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b578f0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57880 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.654+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== osd_map(21..21 src has 1..21) v4 ==== 0+0+0 (unknown 0 0 0) 0x5557e70c3b00 con 0x5557e62f5400 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2bc7e 2019-12-07T19:33:58.654+0000 7f2b834f8700 20 osd.6 21 OSD::ms_dispatch: osd_map(21..21 src has 1..21) v4 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.654+0000 7f2b834f8700 10 osd.6 21 do_waiters -- start 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.654+0000 7f2b834f8700 10 osd.6 21 do_waiters -- finish 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.654+0000 7f2b834f8700 20 osd.6 21 _dispatch 0x5557e70c3b00 osd_map(21..21 src has 1..21) v4 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.654+0000 7f2b834f8700 3 osd.6 21 handle_osd_map epochs [21,21], i have 21, src has [1,21] 2019-12-07T19:33:58.654+0000 7f2b834f8700 10 osd.6 21 no new maps here, dropping 2019-12-07T19:33:58.654+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 aio_wait 2019-12-07T19:33:58.654+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121500 2019-12-07T19:33:58.654+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 io_done 2019-12-07T19:33:58.654+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 aio_wait 2019-12-07T19:33:58.654+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5a300 2019-12-07T19:33:58.654+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 io_done 2019-12-07T19:33:58.654+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 io_done 2019-12-07T19:33:58.654+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 aio_wait 2019-12-07T19:33:58.654+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5800 2019-12-07T19:33:58.654+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 io_done 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 51 ==== MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 909+0+0 (crc 0 0 0) 0x5557e79e3800 con 0x5557e79cac00 2019-12-07T19:33:58.654+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5340 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e767c000 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e75c5340 prio 127 cost 0 latency 0.000061 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'5], crt=21'2) [] 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b578f0 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57880 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.654+0000 7f2b79ce5700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9b57898 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.654+0000 7f2b744da700 20 _add level 1 near 0 on buffer(0x5557e9b59b90 space 0x5557e9b57898 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.654+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 0x0~1000 = 4096 2019-12-07T19:33:58.654+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79f4400 0x5557e75b1b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7692900 seq 10 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.654+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: Getting attr on 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: not in cache 1:3c88320e:::smithi13140796-1:head 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:58.654+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692900 onodes shared_blobs 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: found on disk, size 8192 2019-12-07T19:33:58.654+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 prepare 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692900 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 io_done 2019-12-07T19:33:58.654+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3c88320e:::smithi13140796-1:head# hinfo_key = 0 2019-12-07T19:33:58.654+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e75c5340 finish 2019-12-07T19:33:58.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.654+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 -- 0x5557e9d59400 con 0x5557e62f5400 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7694160 finish 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4c60 prio 127 cost 0 latency 0.000574 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e79f4400 0x5557e75b1b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096]))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,6(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=1(3),5(1),6(0)) 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4c60 finish 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000672134s (0.000000184s flush + 0.000671950s kv commit) 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:33:58.654+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x5557e9d59400 con 0x5557e62f5400 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=3 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.654+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e75c42c0 prio 127 cost 0 latency 0.000095 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 21 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x5557e9c7c000 con 0x5557e79cbc00 2019-12-07T19:33:58.654+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000691 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2081 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:33:58.654+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 52 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:20, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 909+0+0 (crc 0 0 0) 0x5557e763d800 con 0x5557e79cac00 2019-12-07T19:33:58.654+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e76951e0 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:20, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.654+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:20, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb680 had 1 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096, 5(1),4096]))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,6(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=1(3),6(0)) 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3367 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9d58500 had 1 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 2 2019-12-07T19:33:58.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:20, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4972 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:20, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0780 had 1 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 7069 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000708 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.654+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e76951e0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:20, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7d680 had 1 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 1 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8183 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:58.654+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:20, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.654+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8206 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8206 < min_flush_size 524288 2019-12-07T19:33:58.654+0000 7f2b957b2700 10 osd.6 21 ms_handle_authentication new session 0x5557e75589c0 con 0x5557e79f4400 entity osd.5 addr 2019-12-07T19:33:58.654+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~1000 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x2000 (8192) 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'3], crt=21'1) [] 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2bc7e~200e to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7620700 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620690 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.654+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2bc7e 0x2bc7e~200e to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e7620700 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620690 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.654+0000 7f2b957b2700 10 osd.6 21 ms_handle_authentication session 0x5557e75589c0 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xbc7e 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xc7e 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.654+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'2 (0'0) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:31 2019-12-07T19:33:58.635123+0000 0 ObjectCleanRegions clean_offsets: [180224~18446744073709371391], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.654+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xc8c and padding block with 0x374 2019-12-07T19:33:58.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11500> 2019-12-07T19:33:58.654+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (21'1) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:20 2019-12-07T19:33:58.628837+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 kv_submitted 2019-12-07T19:33:58.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11500 2019-12-07T19:33:58.654+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 finishing 2019-12-07T19:33:58.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11500 onodes 2019-12-07T19:33:58.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11500 done 2019-12-07T19:33:58.654+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11500 [] 2019-12-07T19:33:58.654+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.654+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.654+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.654+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.654+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e7a9b400 con 0x5557e7519800 2019-12-07T19:33:58.654+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.654+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.654+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7692600 seq 8 2019-12-07T19:33:58.654+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.654+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.654+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.654+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.654+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.654+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.654+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.654+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692600 onodes shared_blobs 2019-12-07T19:33:58.654+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.654+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 prepare 2019-12-07T19:33:58.654+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692600 2019-12-07T19:33:58.654+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 io_done 2019-12-07T19:33:58.654+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e76951e0 finish 2019-12-07T19:33:58.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2dc8c 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 5 cleaned 0 in 0.000324470s (0.000005285s flush + 0.000319185s kv commit) 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121500,0x5557e9d5a300,0x5557e9b10f00,0x5557e9cc5800,0x5557e7692900> 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121500 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 finishing 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121500 onodes 0x5557e76cb680 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e76cd278 in 0x5557e6269500) _finish_write discard buffer(0x5557e9cd6510 space 0x5557e76cd278 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e76cd978 in 0x5557e6269500) _finish_write discard buffer(0x5557e9cd6480 space 0x5557e76cd978 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a28c58 in 0x5557e6269500) _finish_write discard buffer(0x5557e9cd63f0 space 0x5557e7a28c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a29828 in 0x5557e6269500) _finish_write discard buffer(0x5557e9cd6360 space 0x5557e7a29828 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b574a8 in 0x5557e6269500) _finish_write discard buffer(0x5557e9cd62d0 space 0x5557e9b574a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb3128 in 0x5557e6269500) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9cb3128 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb3978 in 0x5557e6269500) _finish_write discard buffer(0x5557e9cd65a0 space 0x5557e9cb3978 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 done 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2dc8c~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121500 [] 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.655+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2dc8c 0x2dc8c~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 kv_submitted 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5a300 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xdc8c 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xc8c 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xfd and padding block with 0xf03 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 finishing 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5a300 onodes 0x5557e9d58500 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7668b08 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd70e0 space 0x5557e7668b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb3438 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9cb3438 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=2, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e7a9a600 con 0x5557e79ca800 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36a28 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b58c60 space 0x5557e9d36a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36b08 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7170 space 0x5557e9d36b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36be8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7200 space 0x5557e9d36be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36cc8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7290 space 0x5557e9d36cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36da8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9d36da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36e88 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd73b0 space 0x5557e9d36e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a300 done 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5a300 [] 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.655+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10f00 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 finishing 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10f00 onodes 0x5557e9bc0780 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 done 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10f00 [] 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 kv_submitted 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5800 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 finishing 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5800 onodes 0x5557e9b7d680 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c7048 in 0x5557e6269b20) _finish_write discard buffer(0x5557e9cd6bd0 space 0x5557e75c7048 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7668328 in 0x5557e6269b20) _finish_write discard buffer(0x5557e9cd6900 space 0x5557e7668328 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7668ef8 in 0x5557e6269b20) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e7668ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7669128 in 0x5557e6269b20) _finish_write discard buffer(0x5557e9cd6b40 space 0x5557e7669128 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7669198 in 0x5557e6269b20) _finish_write discard buffer(0x5557e9cd6ab0 space 0x5557e7669198 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e76692e8 in 0x5557e6269b20) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e76692e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb32e8 in 0x5557e6269b20) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9cb32e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb3eb8 in 0x5557e6269b20) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9cb3eb8 0x0~1000 writing nocache) 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5800 done 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9e00 q now empty 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5800 [] 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.655+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 21/18 ECSubWriteReply(tid=11, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e7683000 con 0x5557e79cac00 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 kv_submitted 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692900 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 finishing 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692900 onodes 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 done 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692900 [] 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.655+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=18, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cac00 2019-12-07T19:33:58.655+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=4, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e9b36400 con 0x5557e79ca800 2019-12-07T19:33:58.655+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2e0fd 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.655+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.655+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=5, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e9b37a00 con 0x5557e79ca800 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000139401s (0.000000112s flush + 0.000139289s kv commit) 2019-12-07T19:33:58.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.655+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692600> 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 kv_submitted 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692600 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 finishing 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692600 onodes 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692600 done 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692600 [] 2019-12-07T19:33:58.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.655+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=20, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e74f1600 con 0x5557e79cac00 2019-12-07T19:33:58.655+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.655+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e7620700 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620690 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.655+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e76206a8 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.655+0000 7f2b744da700 20 _add level 1 near 0 on buffer(0x5557e9b59560 space 0x5557e76206a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.655+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~1000 = 4096 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.655+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: found on disk, size 8192 2019-12-07T19:33:58.655+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:33:58.655+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:33:58.655+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 -- 0x5557e9c7c500 con 0x5557e62f5400 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4000 finish 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.655+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x5557e9c7c500 con 0x5557e62f5400 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c54a0 prio 127 cost 0 latency 0.001530 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.655+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e75c5a20 prio 127 cost 0 latency 0.000031 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.655+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 5(1),4096]))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,6(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=6(0)) 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c54a0 finish 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4420 prio 127 cost 0 latency 0.001019 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),5(1),6(0)) 2019-12-07T19:33:58.655+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 53 ==== MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 909+0+0 (crc 0 0 0) 0x5557e763d000 con 0x5557e79cac00 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4420 finish 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.655+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84c60 prio 127 cost 0 latency 0.000039 epoch 21 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c42c0 prio 127 cost 0 latency 0.001172 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.655+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.655+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 54 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:32, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9b9e800 con 0x5557e79cac00 2019-12-07T19:33:58.655+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84580 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:32, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.655+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:32, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply Complete: ReadOp(tid=6, to_read={1:3c88320e:::smithi13140796-1:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3c88320e:::smithi13140796-1:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 5(1),4096, 6(0),4096]))}, priority=127, obj_to_source={1:3c88320e:::smithi13140796-1:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3c88320e:::smithi13140796-1:head,4(2)=1:3c88320e:::smithi13140796-1:head,5(1)=1:3c88320e:::smithi13140796-1:head,6(0)=1:3c88320e:::smithi13140796-1:head}, in_progress=) 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] objects_read_async_cb: got: {1:3c88320e:::smithi13140796-1:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.655+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79f4400 0x5557e75b1b80 crc :-1 s=SESSION_ACCEPTING pgs=8 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x5557e79ca000 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 0x5557e9ceea00 2019-12-07T19:33:58.655+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79f4400 0x5557e75b1b80 crc :-1 s=SESSION_ACCEPTING pgs=8 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 1:3c88320e:::smithi13140796-1:head [read 0~1 out=1b] ov 21'4 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op 1:3c88320e:::smithi13140796-1:head [read 0~1 out=1b] 2019-12-07T19:33:58.655+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.655+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx alloc reply 0x5557e9c7c000 result 0 2019-12-07T19:33:58.655+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.655+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] log_op_stats osd_op(client.4540.0:3 1.cs0 1:3c88320e:::smithi13140796-1:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 inb 0 outb 1 lat 0.030540 2019-12-07T19:33:58.656+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.656+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] publish_stats_to_osd 21:18 2019-12-07T19:33:58.656+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(3 smithi13140796-1 [read 0~1 out=1b] v0'0 uv4 ondisk = 0) v8 -- 0x5557e9c7c000 con 0x5557e6f87800 2019-12-07T19:33:58.656+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=READY pgs=8 cs=1 l=0 rx=0 tx=0).ready entity=osd.5 client_cookie=fce9122ddfb0a0db server_cookie=6547c7d58203b459 in_seq=11 out_seq=14 2019-12-07T19:33:58.656+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 18 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:35, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x5557e9c7b000 con 0x5557e7519800 2019-12-07T19:33:58.656+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c42c0 finish 2019-12-07T19:33:58.656+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695340 prio 127 cost 0 latency 0.000035 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:35, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.656+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:35, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.656+0000 7f2b834f8700 2 osd.6 21 ms_handle_reset con 0x5557e79f4400 session 0x5557e75589c0 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5a20 prio 127 cost 0 latency 0.000391 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:35, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:32, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:35, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:32, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 osd.6 21 dequeue_op 0x5557e7695340 prio 127 cost 0 latency 0.000097 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:35, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d84580 prio 127 cost 0 latency 0.000254 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:32, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096, 6(0),4096]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),5(1)) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5a20 finish 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:35, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:32, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:35 2019-12-07T19:33:58.636666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'2) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:32 2019-12-07T19:33:58.635147+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:3f7a742d:::smithi13140796-9:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:26 client_op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[]}) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] repop_all_committed: repop tid 5 all committed 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'2) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:32 2019-12-07T19:33:58.635147+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] eval_repop repgather(0x5557e628b200 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] commit: repgather(0x5557e628b200 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:35 2019-12-07T19:33:58.636666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e766a000 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.656+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] log_op_stats osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 0 outb 0 lat 0.023853 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] sending reply on osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x5557e76ca780 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d84c60 prio 127 cost 0 latency 0.000394 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.656+0000 7f2b77ce1700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(26 smithi13140796-9 [setxattr (55)] v21'6 uv6 ondisk = 0) v8 -- 0x5557e76ca780 con 0x5557e6f87800 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7631200 seq 9 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 3 keys 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9cc4000 seq 7 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 3 keys = 0 2019-12-07T19:33:58.656+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] publish_stats_to_osd 21:19 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:14 2019-12-07T19:33:58.627337+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.656+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 3 keys 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] removing repgather(0x5557e628b200 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 3 keys = 0 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] q front is repgather(0x5557e628b200 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.656+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.656+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.656+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.656+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] remove_repop repgather(0x5557e628b200 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e7a92600 seq 8 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631200 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631200 onodes 0x5557e9c7d180 shared_blobs 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] requeue_op 0x5557e75c49a0 2019-12-07T19:33:58.656+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9c7d288) update 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:1353159e:::smithi13140796-11:head# is 605 (485 bytes onode + 2 bytes spanning blobs + 118 bytes inline extents) 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4000 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4000 onodes 0x5557e9b7ca00 shared_blobs 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557e9b7cb08) update 1#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:a9dea763:::smithi13140796-12:head# is 608 (485 bytes onode + 2 bytes spanning blobs + 121 bytes inline extents) 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 prepare 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92600 onodes shared_blobs 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631200 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] try_finish_rmw: Op(1:3f7a742d:::smithi13140796-9:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:26 client_op=osd_op(client.4540.0:26 1.cs0 1:3f7a742d:::smithi13140796-9:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[]}) 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 io_done 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 prepare 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 prepare 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92600 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4000 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 io_done 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d84580 finish 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: starting commit on Op(1:3f7a742d:::smithi13140796-9:head v=0'0 tt=0'0 tid=8 reqid=client.4540.0:26 roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1645 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7d180 had 1 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1668 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1668 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2e0fd~684 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: op: Op(1:3f7a742d:::smithi13140796-9:head v=0'0 tt=0'0 tid=8 reqid=client.4540.0:26 roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2e0fd 0x2e0fd~684 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xe0fd 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xfd 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x781 and padding block with 0x87f 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e79e3000 con 0x5557e79cac00 2019-12-07T19:33:58.656+0000 7f2b77ce1700 10 osd.6 21 dequeue_op 0x5557e7695340 finish 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 io_done 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=8, reqid=client.4540.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e79e4800 con 0x5557e7519800 2019-12-07T19:33:58.656+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d84c60 finish 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e79e5000 con 0x5557e79ca800 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e763d800 con 0x5557e79cbc00 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e763f000 con 0x5557e79ca000 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2e781 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'3 lcod 21'5 mlcod 21'5 active+clean] append_log log((0'0,21'6], crt=21'3) [] 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000132343s (0.000000046s flush + 0.000132297s kv commit) 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7631200> 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 kv_submitted 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631200 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1648 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 finishing 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] rollforward: entry=21'4 (21'1) modify 1:3c88320e:::smithi13140796-1:head by client.4540.0:2 2019-12-07T19:33:58.625650+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631200 onodes 0x5557e9c7d180 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631200 done 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631200 [] 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.656+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7ca00 had 1 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] rollforward: entry=21'5 (21'2) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:5 2019-12-07T19:33:58.625925+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2762 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=24, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e76b1200 con 0x5557e79cac00 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2785 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2785 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] rollforward: entry=21'6 (21'3) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:26 2019-12-07T19:33:58.630747+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2e781~ae1 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2e781 0x2e781~ae1 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] append_log approx pg log length = 6 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xe781 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x781 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x262 and padding block with 0xd9e 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.656+0000 7f2b784e2700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7a92c00 seq 12 2019-12-07T19:33:58.656+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes shared_blobs 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2f262 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000145646s (0.000000073s flush + 0.000145573s kv commit) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4000,0x5557e7a92600> 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 kv_submitted 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4000 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 finishing 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4000 onodes 0x5557e9b7ca00 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4000 done 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4000 [] 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c49a0 prio 63 cost 0 latency 0.024264 osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2f262~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 kv_submitted 2019-12-07T19:33:58.656+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92600 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2f262 0x2f262~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xf262 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 finishing 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x262 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92600 onodes 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x6d3 and padding block with 0x92d 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92600 done 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92600 [] 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.656+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.656+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e765ea00 con 0x5557e7519800 2019-12-07T19:33:58.656+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op: op osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.656+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=19, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e758da00 con 0x5557e79cac00 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: found obc in cache: 0x5557e9b21680 2019-12-07T19:33:58.656+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(none n=0 w=0) oi: 1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.656+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2f6d3 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op obc obc(1:3f7a742d:::smithi13140796-9:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 0x5557e70b7800 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 1:3f7a742d:::smithi13140796-9:head [read 0~1] ov 21'6 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000180190s (0.000000056s flush + 0.000180134s kv commit) 2019-12-07T19:33:58.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92c00> 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op 1:3f7a742d:::smithi13140796-9:head [read 0~1] 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_read 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] async_read noted for 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:58.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:33:58.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 6(0) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] start_read_op: starting ReadOp(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3f7a742d:::smithi13140796-9:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_read_op: starting read ReadOp(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3f7a742d:::smithi13140796-9:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9c7c500 con 0x5557e79cac00 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e76ca500 con 0x5557e79cbc00 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e7606500 con 0x5557e79ca000 2019-12-07T19:33:58.656+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e7606a00 con 0x5557e62f5400 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_read_op: started ReadOp(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3f7a742d:::smithi13140796-9:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:3f7a742d:::smithi13140796-9:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3f7a742d:::smithi13140796-9:head,4(2)=1:3f7a742d:::smithi13140796-9:head,5(1)=1:3f7a742d:::smithi13140796-9:head,6(0)=1:3f7a742d:::smithi13140796-9:head}, in_progress=1(3),4(2),5(1),6(0)) 2019-12-07T19:33:58.656+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c49a0 finish 2019-12-07T19:33:58.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 12 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 4260+0+0 (unknown 0 0 0) 0x5557e9c7c000 con 0x5557e79ca000 2019-12-07T19:33:58.657+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.657+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x5557e7606a00 con 0x5557e62f5400 2019-12-07T19:33:58.657+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e76954a0 prio 127 cost 0 latency 0.000157 epoch 21 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000050 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:33:58.657+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 19 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e765ea00 con 0x5557e7519800 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096, 5(1),4096, 6(0),4096]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3)) 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84160 finish 2019-12-07T19:33:58.657+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695600 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.657+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e76954a0 prio 127 cost 0 latency 0.000226 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=0,4096,0}, subchunks={1:3f7a742d:::smithi13140796-9:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.657+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.657+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 0x0~1000 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x9000 (36864) 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7621570 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621500 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e7621570 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621500 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 30 ==== MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9b9f000 con 0x5557e79ca800 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d846e0 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.657+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e769f000 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e9d846e0 prio 127 cost 0 latency 0.000136 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e7621570 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e7621518 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 _add level 1 near 0 on buffer(0x5557e79ab440 space 0x5557e7621518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 0x0~1000 = 4096 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:40 2019-12-07T19:33:58.639869+0000 0 ObjectCleanRegions clean_offsets: [229376~18446744073709322239], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_hash_info: Getting attr on 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_hash_info: not in cache 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_hash_info: found on disk, size 36864 2019-12-07T19:33:58.657+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key 2019-12-07T19:33:58.657+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key = 0 2019-12-07T19:33:58.657+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 -- 0x5557e9c7c000 con 0x5557e62f5400 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e7046f00 seq 6 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e76954a0 finish 2019-12-07T19:33:58.657+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:58.657+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x5557e9c7c000 con 0x5557e62f5400 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes shared_blobs 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.657+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e7695b80 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.657+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e9d846e0 finish 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695600 prio 196 cost 0 latency 0.000382 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695600 finish 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695b80 prio 127 cost 0 latency 0.000163 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=9, attrs_read=0) 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2f6d3~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2f6d3 0x2f6d3~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xf6d3 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x6d3 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3f7a742d:::smithi13140796-9:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [6(0),4096]))}, priority=127, obj_to_source={1:3f7a742d:::smithi13140796-9:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3f7a742d:::smithi13140796-9:head,4(2)=1:3f7a742d:::smithi13140796-9:head,5(1)=1:3f7a742d:::smithi13140796-9:head,6(0)=1:3f7a742d:::smithi13140796-9:head}, in_progress=1(3),4(2),5(1)) 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xb44 and padding block with 0x4bc 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695b80 finish 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x2fb44 2019-12-07T19:33:58.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 31 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b37a00 con 0x5557e79ca800 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.657+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84b00 prio 196 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.657+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.657+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000124596s (0.000000092s flush + 0.000124504s kv commit) 2019-12-07T19:33:58.657+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 32 ==== MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e763f000 con 0x5557e79ca800 2019-12-07T19:33:58.657+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d842c0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84b00 prio 196 cost 0 latency 0.000055 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84b00 finish 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 22 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36e00 con 0x5557e79cbc00 2019-12-07T19:33:58.657+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5600 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.657+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e769f000 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5600 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 23 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x5557e7606500 con 0x5557e79cbc00 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e9d842c0 prio 127 cost 0 latency 0.000094 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.657+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000011 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.657+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5600 finish 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000027 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=9, attrs_read=0) 2019-12-07T19:33:58.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 13 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 ==== 4260+0+0 (unknown 0 0 0) 0x5557e9d59b80 con 0x5557e79ca000 2019-12-07T19:33:58.657+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000010 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:41 2019-12-07T19:33:58.639904+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.657+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3f7a742d:::smithi13140796-9:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096, 6(0),4096]))}, priority=127, obj_to_source={1:3f7a742d:::smithi13140796-9:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3f7a742d:::smithi13140796-9:head,4(2)=1:3f7a742d:::smithi13140796-9:head,5(1)=1:3f7a742d:::smithi13140796-9:head,6(0)=1:3f7a742d:::smithi13140796-9:head}, in_progress=1(3),5(1)) 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c58c0 finish 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 14 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9d88800 con 0x5557e79ca000 2019-12-07T19:33:58.657+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4840 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000029 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=9, attrs_read=0) 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:41 2019-12-07T19:33:58.639904+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3f7a742d:::smithi13140796-9:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096, 5(1),4096, 6(0),4096]))}, priority=127, obj_to_source={1:3f7a742d:::smithi13140796-9:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3f7a742d:::smithi13140796-9:head,4(2)=1:3f7a742d:::smithi13140796-9:head,5(1)=1:3f7a742d:::smithi13140796-9:head,6(0)=1:3f7a742d:::smithi13140796-9:head}, in_progress=1(3)) 2019-12-07T19:33:58.657+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5e40 finish 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.657+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.657+0000 7f2b79ce5700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.657+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.657+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:58.657+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.657+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:58.657+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4840 prio 196 cost 0 latency 0.000043 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.657+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:58.657+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 2019-12-07T19:33:58.657+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:58.657+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:58.657+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:58.657+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.657+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e7121800 seq 7 2019-12-07T19:33:58.657+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.657+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4840 finish 2019-12-07T19:33:58.657+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 3 keys 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 3 keys = 0 2019-12-07T19:33:58.657+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.657+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.657+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:58.657+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.658+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:58.658+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121800 onodes 0x5557e76cb680 shared_blobs 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 bluestore.extentmap(0x5557e76cb788) update 2#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 2#1:891f0d13:::smithi13140796-14:head# is 627 (485 bytes onode + 2 bytes spanning blobs + 140 bytes inline extents) 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.658+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 prepare 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121800 2019-12-07T19:33:58.658+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 io_done 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.658+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.658+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e9d842c0 finish 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1212 < min_flush_size 524288 2019-12-07T19:33:58.658+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=6, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e74f0000 con 0x5557e79ca800 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1678 < min_flush_size 524288 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb680 had 1 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:58.658+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1701 < min_flush_size 524288 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1701 < min_flush_size 524288 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x2fb44~6a5 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x2fb44 0x2fb44~6a5 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.627870+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _allocate len 0x1e9 from 0 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _allocate len 0x1e9 from 1 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,acd140000 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd160000~10000/10000,10000,acd140000 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x40000 to include allocated 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 25 (was clean) 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xfb44 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xb44 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x1e9 and padding block with 0xe17 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x40000 2019-12-07T19:33:58.658+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 33 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e763d800 con 0x5557e79ca800 2019-12-07T19:33:58.658+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4b00 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.658+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4b00 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.6s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:47, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'5], crt=21'2) [21'6 (21'4) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:47 2019-12-07T19:33:58.643819+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'2 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'4) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:47 2019-12-07T19:33:58.643819+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] rollforward: entry=21'3 (20'1) modify 1:657ac0fe:::smithi13140796-4:head by client.4540.0:11 2019-12-07T19:33:58.626793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] rollforward: entry=21'4 (0'0) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:46 2019-12-07T19:33:58.643677+0000 0 ObjectCleanRegions clean_offsets: [262144~18446744073709289471], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.658+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x301e9 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7630000 seq 10 2019-12-07T19:33:58.658+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 3 keys 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 3 keys = 0 2019-12-07T19:33:58.658+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.658+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630000 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630000 onodes 0x5557e9d58500 shared_blobs 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9d58608) update 1#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (25) on file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]), flushing log 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:6336c6f2:::smithi13140796-16:head# is 646 (485 bytes onode + 2 bytes spanning blobs + 159 bytes inline extents) 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 25 len 0x48 crc 0x7ffde0f4) 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 prepare 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x18000~1000 to file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630000 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x18000 0x18000~1000 to file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 io_done 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x18000 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.658+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4b00 finish 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x19000 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000018 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 25 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000462154s (0.000000070s flush + 0.000462084s kv commit) 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121800> 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 kv_submitted 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121800 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1686 < min_flush_size 524288 2019-12-07T19:33:58.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 finishing 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121800 onodes 0x5557e76cb680 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121800 done 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:58.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121800 [] 2019-12-07T19:33:58.658+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9d58500 had 1 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.658+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1709 < min_flush_size 524288 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1709 < min_flush_size 524288 2019-12-07T19:33:58.658+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=7, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9b37000 con 0x5557e79ca800 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x301e9~6ad to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x301e9 0x301e9~6ad to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x1e9 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x1e9 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x896 and padding block with 0x76a 2019-12-07T19:33:58.658+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x30896 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.658+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000101047s (0.000000063s flush + 0.000100984s kv commit) 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630000> 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 kv_submitted 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630000 2019-12-07T19:33:58.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 finishing 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630000 onodes 0x5557e9d58500 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630000 done 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630000 [] 2019-12-07T19:33:58.658+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.658+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.658+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e74f1000 con 0x5557e79ca800 2019-12-07T19:33:58.658+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.659+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 34 ==== MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (crc 0 0 0) 0x5557e9d59900 con 0x5557e79ca800 2019-12-07T19:33:58.659+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5080 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.659+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c5080 prio 127 cost 0 latency 0.000144 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=9, to_read={1:6c205177:::smithi13140796-8:head=0,4096,0}, subchunks={1:6c205177:::smithi13140796-8:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.659+0000 7f2b734d8700 25 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.659+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 0x0~1000 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x8000 (32768) 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e76210a0 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2e70 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e76210a0 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2e70 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e76210a0 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2e70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9be2e88 in 0x5557e6268fc0) _discard 0x0~1000 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 _add level 1 near 0 on buffer(0x5557e9cd7440 space 0x5557e9be2e88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 0x0~1000 = 4096 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:6c205177:::smithi13140796-8:head 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:6c205177:::smithi13140796-8:head 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 32768 2019-12-07T19:33:58.659+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# hinfo_key 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:6c205177:::smithi13140796-8:head# hinfo_key = 0 2019-12-07T19:33:58.659+0000 7f2b734d8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 -- 0x5557e9bc0000 con 0x5557e79ca800 2019-12-07T19:33:58.659+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c5080 finish 2019-12-07T19:33:58.659+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.660+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 35 ==== MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (crc 0 0 0) 0x5557e9bc0000 con 0x5557e79ca800 2019-12-07T19:33:58.660+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.660+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.660+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 24 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 71604+0+0 (crc 0 0 0) 0x5557e79e5000 con 0x5557e79cbc00 2019-12-07T19:33:58.660+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c51e0 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.660+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c51e0 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:49, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:49 2019-12-07T19:33:58.656740+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:49 2019-12-07T19:33:58.656740+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7692c00 seq 5 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.3s5_head 0x5557e75b4000) get_onode oid 5#1:c87708e9:::smithi13140796-17:head# key 0x858000000000000001c87708e921736d'ithi13140796-17!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.660+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1100 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# = 0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# 2 keys 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# 2 keys = 0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# 0x0~11000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:c87708e9:::smithi13140796-17:head# 0x0~11000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~11000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692c00 9 blobs 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate 0x11000/1000,11000,0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xc0000~11000/1000,11000,0 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xc0000~11000] 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d922a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92310 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d922a0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92310) get_ref 0x0~2000 Blob(0x5557e9d92310 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d922a0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9d92310 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d922a0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d922b8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668690 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36540 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e7668690) get_ref 0x0~2000 Blob(0x5557e7668690 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36540 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e7668690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7668690 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36540 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d36558 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d9ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d923f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d9ea0 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d923f0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e74d9ea0) get_ref 0x0~2000 Blob(0x5557e74d9ea0 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d923f0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e74d9ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e74d9ea0 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d923f0 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d92408 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92070 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92460 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92070) get_ref 0x0~2000 Blob(0x5557e9d92070 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92460 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9d92070 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92460 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d92478 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d924d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d924d0 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92540 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d924d0) get_ref 0x0~2000 Blob(0x5557e9d924d0 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92540 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d924d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9d924d0 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92540 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d92558 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d925b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d925b0 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92620 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d925b0) get_ref 0x0~2000 Blob(0x5557e9d925b0 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92620 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d925b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9d925b0 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92620 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d92638 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92690 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92700 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92690) get_ref 0x0~2000 Blob(0x5557e9d92690 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92700 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9d92690 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92700 sbid 0x0)) 2019-12-07T19:33:58.660+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d92718 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d927e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92770 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d927e0 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92770) get_ref 0x0~2000 Blob(0x5557e9d92770 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d927e0 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9d92770 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d927e0 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d927f8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.661+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 55 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x5557e9d59680 con 0x5557e79cac00 2019-12-07T19:33:58.661+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5340 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d928c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.661+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92850 blob([0xd0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d928c0 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92850) get_ref 0x0~1000 Blob(0x5557e9d92850 blob([0xd0000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d928c0 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92850) get_ref init 0x1000, 1000 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~1000: 0x0~1000 Blob(0x5557e9d92850 blob([0xd0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9d928c0 sbid 0x0)) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d928d8 in 0x5557e62696c0) _discard 0x0~1000 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x11000 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cb008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.661+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# 0x0~11000 = 0 2019-12-07T19:33:58.661+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.661+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.661+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 56 ==== MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e79e3000 con 0x5557e79cac00 2019-12-07T19:33:58.661+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5a20 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.661+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.661+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.661+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 57 ==== MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9b9e000 con 0x5557e79cac00 2019-12-07T19:33:58.661+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c46e0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.661+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e767c000 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e75c5a20 prio 127 cost 0 latency 0.000077 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.661+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5340 prio 127 cost 0 latency 0.000161 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.661+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.661+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.661+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e7664900 seq 11 2019-12-07T19:33:58.661+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# 3 keys 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply Complete: ReadOp(tid=7, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 5(1),4096, 6(0),4096]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=) 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# 3 keys = 0 2019-12-07T19:33:58.661+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.9s5_head 5#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.661+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692c00 cost 112062 (10 ios * 4000 + 72062 bytes) 2019-12-07T19:33:58.661+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692c00 onodes 0x5557e76caf00 shared_blobs 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] objects_read_async_cb: got: {1:33f5bf01:::smithi13140796-2:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cb008) update 5#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cb008) update inline shard 171 bytes from 9 extents 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:c87708e9:::smithi13140796-17:head# is 590 (413 bytes onode + 2 bytes spanning blobs + 175 bytes inline extents) 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.661+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 0x5557e70b6a00 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692c00 allocated 0x[c0000~11000] released 0x[] 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:58.661+0000 7f2b78ce3700 10 freelist allocate 0xc0000~11000 2019-12-07T19:33:58.661+0000 7f2b78ce3700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664900 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664900 onodes 0x5557e9b7c000 shared_blobs 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e9b7c108) update 5#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.661+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~1 out=1b] ov 21'5 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:9da56077:::smithi13140796-13:head# is 624 (485 bytes onode + 2 bytes spanning blobs + 137 bytes inline extents) 2019-12-07T19:33:58.661+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 prepare 2019-12-07T19:33:58.661+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692c00 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.661+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~1 out=1b] 2019-12-07T19:33:58.661+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 prepare 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664900 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 io_done 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx alloc reply 0x5557e9d59b80 result 0 2019-12-07T19:33:58.661+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.661+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e75c5a20 finish 2019-12-07T19:33:58.661+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.661+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] log_op_stats osd_op(client.4540.0:6 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e20) v8 inb 0 outb 1 lat 0.035768 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.661+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] publish_stats_to_osd 21:20 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1664 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(6 smithi13140796-2 [read 0~1 out=1b] v0'0 uv5 ondisk = 0) v8 -- 0x5557e9d59b80 con 0x5557e6f87800 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7c000 had 1 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:58.661+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5340 finish 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1687 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1687 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x30896~697 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x30896 0x30896~697 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e766d000 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x896 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x896 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xf2d and padding block with 0xd3 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e75c46e0 prio 127 cost 0 latency 0.000315 MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:43 2019-12-07T19:33:58.641743+0000 0 ObjectCleanRegions clean_offsets: [245760~18446744073709305855], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.661+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e9d5b500 seq 6 2019-12-07T19:33:58.661+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5b500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5b500 onodes shared_blobs 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5b500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 prepare 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5b500 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 io_done 2019-12-07T19:33:58.661+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e75c46e0 finish 2019-12-07T19:33:58.661+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x30f2d 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000548648s (0.000000074s flush + 0.000548574s kv commit) 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 1 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x30f2d~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.661+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x30f2d 0x30f2d~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xf2d 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xf2d 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.661+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x39e and padding block with 0xc62 2019-12-07T19:33:58.661+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.661+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7664900> 2019-12-07T19:33:58.661+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.661+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 kv_submitted 2019-12-07T19:33:58.661+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664900 2019-12-07T19:33:58.661+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 finishing 2019-12-07T19:33:58.661+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664900 onodes 0x5557e9b7c000 2019-12-07T19:33:58.661+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664900 done 2019-12-07T19:33:58.661+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 58 ==== MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9ba1000 con 0x5557e79cac00 2019-12-07T19:33:58.661+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:58.661+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664900 [] 2019-12-07T19:33:58.661+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.661+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5760 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.661+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.661+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.661+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 59 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=28, reqid=client.4540.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9b9f800 con 0x5557e79cac00 2019-12-07T19:33:58.661+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=25, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e9b36400 con 0x5557e79cac00 2019-12-07T19:33:58.662+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695ce0 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=28, reqid=client.4540.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.662+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=28, reqid=client.4540.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.662+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=28, reqid=client.4540.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.662+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=28, reqid=client.4540.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.662+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e7695ce0 prio 127 cost 0 latency 0.000045 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=28, reqid=client.4540.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.662+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=28, reqid=client.4540.0:32, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.662+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.662+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'2) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:32 2019-12-07T19:33:58.635147+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.662+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.662+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.662+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.662+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.662+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.662+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c51e0 finish 2019-12-07T19:33:58.662+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.662+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7693b00 seq 10 2019-12-07T19:33:58.662+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.662+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.662+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.662+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.662+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.662+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.662+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693b00 onodes shared_blobs 2019-12-07T19:33:58.662+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.662+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 prepare 2019-12-07T19:33:58.662+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693b00 2019-12-07T19:33:58.662+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 io_done 2019-12-07T19:33:58.662+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e7695ce0 finish 2019-12-07T19:33:58.662+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 20 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:35, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (unknown 0 0 0) 0x5557e79e4800 con 0x5557e7519800 2019-12-07T19:33:58.662+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:35, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.662+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:35, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.662+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 60 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x5557e9d59b80 con 0x5557e79cac00 2019-12-07T19:33:58.662+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.662+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 21 ==== MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=4, to_read={1:a9dea763:::smithi13140796-12:head=0,4096,0}, subchunks={1:a9dea763:::smithi13140796-12:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x5557e7606280 con 0x5557e7519800 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.662+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 aio_wait 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=4, to_read={1:a9dea763:::smithi13140796-12:head=0,4096,0}, subchunks={1:a9dea763:::smithi13140796-12:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.662+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692c00 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.662+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=4, to_read={1:a9dea763:::smithi13140796-12:head=0,4096,0}, subchunks={1:a9dea763:::smithi13140796-12:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.662+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 io_done 2019-12-07T19:33:58.662+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000043 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.662+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.662+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=9, attrs_read=0) 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:35, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:35, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000104 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:35, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:35, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply Complete: ReadOp(tid=9, to_read={1:3f7a742d:::smithi13140796-9:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3f7a742d:::smithi13140796-9:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 5(1),4096, 6(0),4096]))}, priority=127, obj_to_source={1:3f7a742d:::smithi13140796-9:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3f7a742d:::smithi13140796-9:head,4(2)=1:3f7a742d:::smithi13140796-9:head,5(1)=1:3f7a742d:::smithi13140796-9:head,6(0)=1:3f7a742d:::smithi13140796-9:head}, in_progress=) 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:35 2019-12-07T19:33:58.636666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] objects_read_async_cb: got: {1:3f7a742d:::smithi13140796-9:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.662+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.662+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 0x5557e70b7800 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.662+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 1:3f7a742d:::smithi13140796-9:head [read 0~1 out=1b] ov 21'6 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9cc5b00 seq 8 2019-12-07T19:33:58.662+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.662+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op 1:3f7a742d:::smithi13140796-9:head [read 0~1 out=1b] 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.662+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx alloc reply 0x5557e7606a00 result 0 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5b00 onodes shared_blobs 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.662+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5b00 prepare 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5b00 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5b00 io_done 2019-12-07T19:33:58.662+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] log_op_stats osd_op(client.4540.0:27 1.cs0 1:3f7a742d:::smithi13140796-9:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 1 lat 0.030030 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4f20 finish 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=4, to_read={1:a9dea763:::smithi13140796-12:head=0,4096,0}, subchunks={1:a9dea763:::smithi13140796-12:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.662+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] publish_stats_to_osd 21:21 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(27 smithi13140796-9 [read 0~1 out=1b] v0'0 uv6 ondisk = 0) v8 -- 0x5557e7606a00 con 0x5557e6f87800 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=4, to_read={1:a9dea763:::smithi13140796-12:head=0,4096,0}, subchunks={1:a9dea763:::smithi13140796-12:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000214 MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=4, to_read={1:a9dea763:::smithi13140796-12:head=0,4096,0}, subchunks={1:a9dea763:::smithi13140796-12:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.662+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7694b00 finish 2019-12-07T19:33:58.662+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=4, to_read={1:a9dea763:::smithi13140796-12:head=0,4096,0}, subchunks={1:a9dea763:::smithi13140796-12:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.662+0000 7f2b73cd9700 25 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.662+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 0x0~1000 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0xc000 (49152) 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c6ee0 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb25b0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e75c6ee0 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb25b0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.662+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e766d000 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e75c5760 prio 127 cost 0 latency 0.000621 MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:44 2019-12-07T19:33:58.641780+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:44 2019-12-07T19:33:58.641780+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.662+0000 7f2b794e4700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e9b10f00 seq 7 2019-12-07T19:33:58.662+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 3 keys 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 3 keys = 0 2019-12-07T19:33:58.662+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.662+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10f00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10f00 onodes 0x5557e9b7d680 shared_blobs 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore.extentmap(0x5557e9b7d788) update 1#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:51ccf92f:::smithi13140796-15:head# is 643 (485 bytes onode + 2 bytes spanning blobs + 156 bytes inline extents) 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 prepare 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 io_done 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.662+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e75c5760 finish 2019-12-07T19:33:58.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.662+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.002634 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.662+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=10, to_read={1:657ac0fe:::smithi13140796-4:head=0,4096,0}, subchunks={1:657ac0fe:::smithi13140796-4:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.662+0000 7f2b774e0700 25 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.662+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 0x0~1000 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x4000 (16384) 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7a28a80 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621030 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e7a28a80 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621030 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.662+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.662+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 25 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 75700+0+0 (crc 0 0 0) 0x5557e7a11000 con 0x5557e79cbc00 2019-12-07T19:33:58.662+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695a20 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.662+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.662+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.662+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.662+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.662+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e7695a20 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.662+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.662+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.662+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.662+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.662+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.662+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.662+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7630300 seq 5 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.bs1_head 0x5557e75b41e0) get_onode oid 1#1:db85c185:::smithi13140796-18:head# key 0x818000000000000001db85c18521736d'ithi13140796-18!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.663+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1101 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# = 0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~12000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7630300 9 blobs 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate 0x12000/1000,12000,0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xd1000~12000/1000,12000,0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xd1000~12000] 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6e00 blob([0xd1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2930 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e75c6e00) get_ref 0x0~2000 Blob(0x5557e75c6e00 blob([0xd1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2930 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e75c6e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e75c6e00 blob([0xd1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2930 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9be2948 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c6000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c72d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c6000 blob([0xd3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c72d0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e75c6000) get_ref 0x0~2000 Blob(0x5557e75c6000 blob([0xd3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c72d0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e75c6000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e75c6000 blob([0xd3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c72d0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e75c72e8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d99d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be37a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d99d0 blob([0xd5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be37a0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74d99d0) get_ref 0x0~2000 Blob(0x5557e74d99d0 blob([0xd5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be37a0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74d99d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e74d99d0 blob([0xd5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be37a0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9be37b8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be36c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be36c0 blob([0xd7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37dc0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9be36c0) get_ref 0x0~2000 Blob(0x5557e9be36c0 blob([0xd7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37dc0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9be36c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9be36c0 blob([0xd7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37dc0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d37dd8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7668460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668770 blob([0xd9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668460 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7668770) get_ref 0x0~2000 Blob(0x5557e7668770 blob([0xd9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7668460 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7668770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e7668770 blob([0xd9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7668460 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e7668478 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76696c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76696c0 blob([0xdb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621730 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e76696c0) get_ref 0x0~2000 Blob(0x5557e76696c0 blob([0xdb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621730 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e76696c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e76696c0 blob([0xdb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621730 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e7621748 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7620540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e76216c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7620540 blob([0xdd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76216c0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7620540) get_ref 0x0~2000 Blob(0x5557e7620540 blob([0xdd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e76216c0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7620540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e7620540 blob([0xdd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76216c0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e76216d8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b56e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b56e70 blob([0xdf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57a40 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b56e70) get_ref 0x0~2000 Blob(0x5557e9b56e70 blob([0xdf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b57a40 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b56e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9b56e70 blob([0xdf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57a40 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9b57a58 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be3730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be3730 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3810 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9be3730) get_ref 0x0~2000 Blob(0x5557e9be3730 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be3810 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9be3730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9be3730 blob([0xe1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be3810 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9be3828 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x12000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9c7c108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 = 0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630300 cost 116158 (10 ios * 4000 + 76158 bytes) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630300 onodes 0x5557e9c7c000 shared_blobs 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9c7c108) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9c7c108) update inline shard 174 bytes from 9 extents 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 593 (413 bytes onode + 2 bytes spanning blobs + 178 bytes inline extents) 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630300 allocated 0x[d1000~12000] released 0x[] 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 freelist allocate 0xd1000~12000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 prepare 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7630300 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.663+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e7695a20 finish 2019-12-07T19:33:58.663+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.663+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 aio_wait 2019-12-07T19:33:58.663+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630300 2019-12-07T19:33:58.663+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 io_done 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3139e 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.001926125s (0.000000069s flush + 0.001926056s kv commit) 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3230 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e75c6ee0 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb25b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.663+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9cb25c8 in 0x5557e6269180) _discard 0x0~1000 2019-12-07T19:33:58.663+0000 7f2b73cd9700 20 _add level 1 near 0 on buffer(0x5557e79ab320 space 0x5557e9cb25c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76caf00 had 1 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.663+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 0x0~1000 = 4096 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4344 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6027 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7d680 had 1 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 1 2019-12-07T19:33:58.663+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:a9dea763:::smithi13140796-12:head 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8146 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:a9dea763:::smithi13140796-12:head 2019-12-07T19:33:58.663+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7c000 had 1 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8169 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 49152 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8169 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# hinfo_key 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.663+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.5s1_head 1#1:a9dea763:::smithi13140796-12:head# hinfo_key = 0 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3139e~1fe9 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.663+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3139e 0x3139e~1fe9 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.663+0000 7f2b73cd9700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.5s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 -- 0x5557e9b7d180 con 0x5557e7519800 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x139e 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x39e 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.663+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x387 and padding block with 0xc79 2019-12-07T19:33:58.663+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.663+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5b500> 2019-12-07T19:33:58.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 kv_submitted 2019-12-07T19:33:58.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5b500 2019-12-07T19:33:58.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 finishing 2019-12-07T19:33:58.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5b500 onodes 2019-12-07T19:33:58.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5b500 done 2019-12-07T19:33:58.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5b500 [] 2019-12-07T19:33:58.664+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.664+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e7a28a80 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621030 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.664+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e7621048 in 0x5557e6268fc0) _discard 0x0~1000 2019-12-07T19:33:58.664+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 21/18 ECSubWriteReply(tid=26, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e9b36800 con 0x5557e79cac00 2019-12-07T19:33:58.664+0000 7f2b774e0700 20 _add level 1 near 0 on buffer(0x5557e9cd74d0 space 0x5557e7621048 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.664+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 0x0~1000 = 4096 2019-12-07T19:33:58.664+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.664+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:657ac0fe:::smithi13140796-4:head 2019-12-07T19:33:58.664+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:657ac0fe:::smithi13140796-4:head 2019-12-07T19:33:58.664+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# 2019-12-07T19:33:58.664+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 16384 2019-12-07T19:33:58.664+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# hinfo_key 2019-12-07T19:33:58.664+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# hinfo_key = 0 2019-12-07T19:33:58.664+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=10, attrs_read=0)) v2 -- 0x5557e9d58280 con 0x5557e79ca800 2019-12-07T19:33:58.664+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c5b80 finish 2019-12-07T19:33:58.664+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 36 ==== MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e7a10800 con 0x5557e79ca800 2019-12-07T19:33:58.664+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e76954a0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.664+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.664+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 37 ==== MOSDECSubOpRead(1.1s2 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9d58280 con 0x5557e79ca800 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.664+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7694160 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpRead(1.1s2 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.664+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpRead(1.1s2 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e769f000 2019-12-07T19:33:58.664+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.664+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpRead(1.1s2 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.664+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 61 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c76c00 con 0x5557e79cac00 2019-12-07T19:33:58.664+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695760 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.664+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e76954a0 prio 127 cost 0 latency 0.000073 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.664+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.664+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.664+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:41 2019-12-07T19:33:58.639904+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.664+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.664+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695760 prio 196 cost 0 latency 0.000075 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.664+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=8, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.664+0000 7f2b79ce5700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.664+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_finish_rmw: Op(1:3f7a742d:::smithi13140796-9:head v=0'0 tt=0'0 tid=8 reqid=client.4540.0:26 roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.664+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e9b11b00 seq 8 2019-12-07T19:33:58.664+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:33:58.664+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695760 finish 2019-12-07T19:33:58.664+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:58.664+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11b00 onodes shared_blobs 2019-12-07T19:33:58.664+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.664+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 prepare 2019-12-07T19:33:58.664+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 62 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 79796+0+0 (crc 0 0 0) 0x5557e7a0e000 con 0x5557e79cac00 2019-12-07T19:33:58.665+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11b00 2019-12-07T19:33:58.665+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 io_done 2019-12-07T19:33:58.665+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e76954a0 finish 2019-12-07T19:33:58.665+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.665+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84000 prio 127 cost 0 latency 0.000049 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.665+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.665+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpRead(1.1s2 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e769f000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.665+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e7694160 prio 127 cost 0 latency 0.000296 MOSDECSubOpRead(1.1s2 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.665+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.1s2 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d84000 prio 127 cost 0 latency 0.000106 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.665+0000 7f2b75cdd700 25 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.665+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 0x0~1000 2019-12-07T19:33:58.665+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0xe000 (57344) 2019-12-07T19:33:58.665+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.665+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b570a0 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3110 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.665+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b570a0 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3110 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.665+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.665+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e9cc4600 seq 9 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.7s1_head 0x5557e76a92c0) get_onode oid 1#1:ec1285ab:::smithi13140796-19:head# key 0x818000000000000001ec1285ab21736d'ithi13140796-19!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.665+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1102 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# = 0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~13000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9cc4600 10 blobs 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate 0x13000/1000,13000,0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xe3000~13000/1000,13000,0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xe3000~13000] 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7621dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7621dc0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2540 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7621dc0) get_ref 0x0~2000 Blob(0x5557e7621dc0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2540 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7621dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7621dc0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2540 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9be2558 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76681c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7620b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76681c0 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620b60 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e76681c0) get_ref 0x0~2000 Blob(0x5557e76681c0 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7620b60 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e76681c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e76681c0 blob([0xe5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620b60 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e7620b78 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7621ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7621ea0 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621f10 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7621ea0) get_ref 0x0~2000 Blob(0x5557e7621ea0 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621f10 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7621ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e7621ea0 blob([0xe7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621f10 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e7621f28 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7621f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7621f80 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93490 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7621f80) get_ref 0x0~2000 Blob(0x5557e7621f80 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93490 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e7621f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e7621f80 blob([0xe9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93490 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d934a8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d93500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d93500 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93570 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93500) get_ref 0x0~2000 Blob(0x5557e9d93500 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93570 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9d93500 blob([0xeb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93570 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d93588 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d935e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d935e0 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93650 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d935e0) get_ref 0x0~2000 Blob(0x5557e9d935e0 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93650 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d935e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9d935e0 blob([0xed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93650 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d93668 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d936c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d936c0 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93730 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d936c0) get_ref 0x0~2000 Blob(0x5557e9d936c0 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93730 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d936c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9d936c0 blob([0xef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93730 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d93748 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d937a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d937a0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93810 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d937a0) get_ref 0x0~2000 Blob(0x5557e9d937a0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93810 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d937a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9d937a0 blob([0xf1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93810 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d93828 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d93880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d938f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d93880 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d938f0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93880) get_ref 0x0~2000 Blob(0x5557e9d93880 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d938f0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9d93880 blob([0xf3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d938f0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d93908 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d93960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d939d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d93960 blob([0xf5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d939d0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93960) get_ref 0x0~1000 Blob(0x5557e9d93960 blob([0xf5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d939d0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93960) get_ref init 0x1000, 1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~1000: 0x0~1000 Blob(0x5557e9d93960 blob([0xf5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9d939d0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d939e8 in 0x5557e6269880) _discard 0x0~1000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x13000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9c7c888) dirty_range mark inline shard dirty 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 = 0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4600 cost 123446 (11 ios * 4000 + 79446 bytes) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4600 onodes 0x5557e9c7c780 shared_blobs 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9c7c888) update 1#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9c7c888) update inline shard 190 bytes from 10 extents 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:ec1285ab:::smithi13140796-19:head# is 609 (413 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4600 allocated 0x[e3000~13000] released 0x[] 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 freelist allocate 0xe3000~13000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 prepare 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9cc4600 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.665+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d84000 finish 2019-12-07T19:33:58.665+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.665+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 63 ==== MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e7a10000 con 0x5557e79cac00 2019-12-07T19:33:58.665+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84c60 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.665+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.665+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.665+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGOpItem(op=MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e767c000 2019-12-07T19:33:58.665+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e9d84c60 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.665+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s5 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.666+0000 7f2b79ce5700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e9b11800 seq 12 2019-12-07T19:33:58.666+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11800 onodes shared_blobs 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 prepare 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11800 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 io_done 2019-12-07T19:33:58.666+0000 7f2b79ce5700 10 osd.6 21 dequeue_op 0x5557e9d84c60 finish 2019-12-07T19:33:58.666+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 aio_wait 2019-12-07T19:33:58.666+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4600 2019-12-07T19:33:58.666+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 io_done 2019-12-07T19:33:58.666+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b570a0 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3110 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.666+0000 7f2b75cdd700 20 bluestore.BufferSpace(0x5557e9cb3128 in 0x5557e6269500) _discard 0x0~1000 2019-12-07T19:33:58.666+0000 7f2b75cdd700 20 _add level 1 near 0 on buffer(0x5557e9b58d80 space 0x5557e9cb3128 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.666+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 0x0~1000 = 4096 2019-12-07T19:33:58.666+0000 7f2b75cdd700 20 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.666+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:33:58.666+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:33:58.666+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.666+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 57344 2019-12-07T19:33:58.666+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# hinfo_key 2019-12-07T19:33:58.666+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# hinfo_key = 0 2019-12-07T19:33:58.666+0000 7f2b75cdd700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 21/18 ECSubReadReply(tid=12, attrs_read=0)) v2 -- 0x5557e76cbb80 con 0x5557e79ca800 2019-12-07T19:33:58.666+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e7694160 finish 2019-12-07T19:33:58.666+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x33387 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 5 cleaned 0 in 0.003057601s (0.000004720s flush + 0.003052881s kv commit) 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2228 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3549 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7c780 had 1 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3572 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3572 < min_flush_size 524288 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x33387~df4 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.666+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x33387 0x33387~df4 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x3387 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x387 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.666+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x17b and padding block with 0xe85 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693b00,0x5557e7692c00,0x5557e9cc5b00,0x5557e9b10f00,0x5557e7630300> 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693b00 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 finishing 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693b00 onodes 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693b00 done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693b00 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692c00 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 finishing 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692c00 onodes 0x5557e76caf00 2019-12-07T19:33:58.667+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36558 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79ab170 space 0x5557e9d36558 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d922b8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9b59e60 space 0x5557e9d922b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92408 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9b59ef0 space 0x5557e9d92408 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=28, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e7683c00 con 0x5557e79cac00 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92478 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9b58c60 space 0x5557e9d92478 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92558 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79aac60 space 0x5557e9d92558 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92638 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79aa360 space 0x5557e9d92638 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92718 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79aa990 space 0x5557e9d92718 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d927f8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79ab950 space 0x5557e9d927f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d928d8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd6000 space 0x5557e9d928d8 0x0~1000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e794f000 con 0x5557e79cbc00 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692c00 done 2019-12-07T19:33:58.667+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692c00 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5b00 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5b00 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5b00 finishing 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5b00 onodes 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5b00 done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5b00 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10f00 2019-12-07T19:33:58.667+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e765e600 con 0x5557e7519800 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 finishing 2019-12-07T19:33:58.667+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10f00 onodes 0x5557e9b7d680 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9e00 q now empty 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10f00 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630300 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 finishing 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630300 onodes 0x5557e9c7c000 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c72e8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9b58990 space 0x5557e75c72e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e76216d8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e76216d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621748 in 0x5557e6269c00) _finish_write discard buffer(0x5557e79aa120 space 0x5557e7621748 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 21/18 ECSubWriteReply(tid=27, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9c77800 con 0x5557e79cac00 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7668478 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9b595f0 space 0x5557e7668478 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b57a58 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9b590e0 space 0x5557e9b57a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be2948 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9b58b40 space 0x5557e9be2948 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be37b8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9be37b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e74f0000 con 0x5557e79cbc00 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be3828 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9b59cb0 space 0x5557e9be3828 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37dd8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9b59680 space 0x5557e9d37dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630300 done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.667+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630300 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.667+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3417b 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 3 cleaned 0 in 0.000361325s (0.000004459s flush + 0.000356866s kv commit) 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11b00,0x5557e9b11800,0x5557e9cc4600> 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11b00 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 finishing 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11b00 onodes 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11b00 done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11b00 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11800 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 finishing 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11800 onodes 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11800 done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11800 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4600 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 finishing 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4600 onodes 0x5557e9c7c780 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7620b78 in 0x5557e6269880) _finish_write discard buffer(0x5557e79ab680 space 0x5557e7620b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621f28 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e7621f28 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be2558 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aaea0 space 0x5557e9be2558 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d934a8 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd65a0 space 0x5557e9d934a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93588 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd70e0 space 0x5557e9d93588 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93668 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9d93668 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93748 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9d93748 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93828 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7710 space 0x5557e9d93828 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93908 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd77a0 space 0x5557e9d93908 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d939e8 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7830 space 0x5557e9d939e8 0x0~1000 writing nocache) 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4600 done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4600 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.667+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=30, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e76b1e00 con 0x5557e79cac00 2019-12-07T19:33:58.667+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=11, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e758da00 con 0x5557e79ca800 2019-12-07T19:33:58.667+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=31, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e62b8800 con 0x5557e79cac00 2019-12-07T19:33:58.667+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 64 ==== MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e2000 con 0x5557e79cac00 2019-12-07T19:33:58.667+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d846e0 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.667+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e766d000 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d846e0 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.as1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:44 2019-12-07T19:33:58.641780+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.667+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e9d5b800 seq 8 2019-12-07T19:33:58.667+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5b800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5b800 onodes shared_blobs 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5b800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 prepare 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5b800 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 io_done 2019-12-07T19:33:58.667+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d846e0 finish 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.667+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 1 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3417b~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3417b 0x3417b~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x417b 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x17b 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x5ec and padding block with 0xa14 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x345ec 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.667+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000101775s (0.000000066s flush + 0.000101709s kv commit) 2019-12-07T19:33:58.667+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5b800> 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 kv_submitted 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5b800 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 finishing 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5b800 onodes 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5b800 done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9e00 q now empty 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5b800 [] 2019-12-07T19:33:58.667+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.667+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.667+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 21/18 ECSubWriteReply(tid=33, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9b37000 con 0x5557e79cac00 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 38 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e4000 con 0x5557e79ca800 2019-12-07T19:33:58.668+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d849a0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.668+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 39 ==== MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e76cbb80 con 0x5557e79ca800 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.668+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84580 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.668+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d849a0 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:47, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'6], crt=21'4) [] 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'5 (21'2) modify 1:6c205177:::smithi13140796-8:head by client.4540.0:23 2019-12-07T19:33:58.629675+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'4) modify 1:6336c6f2:::smithi13140796-16:head by client.4540.0:47 2019-12-07T19:33:58.643819+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.668+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7631b00 seq 11 2019-12-07T19:33:58.668+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631b00 onodes shared_blobs 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 prepare 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631b00 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 io_done 2019-12-07T19:33:58.668+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d849a0 finish 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e9d84580 prio 127 cost 0 latency 0.000202 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=15, to_read={1:6336c6f2:::smithi13140796-16:head=0,4096,0}, subchunks={1:6336c6f2:::smithi13140796-16:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.668+0000 7f2b774e0700 25 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.668+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 0x0~1000 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x10000 (65536) 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7668b60 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3420 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e7668b60 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3420 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.668+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 65 ==== MOSDECSubOpRead(1.as1 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e7606a00 con 0x5557e79cac00 2019-12-07T19:33:58.668+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84f20 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpRead(1.as1 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.668+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpRead(1.as1 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpRead(1.as1 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGOpItem(op=MOSDECSubOpRead(1.as1 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e766d000 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e9d84f20 prio 127 cost 0 latency 0.000075 MOSDECSubOpRead(1.as1 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.as1 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.668+0000 7f2b794e4700 25 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.668+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 0x0~1000 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0xf000 (61440) 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7668620 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb32d0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e7668620 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb32d0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x345ec~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x345ec 0x345ec~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x45ec 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x5ec 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa5d and padding block with 0x5a3 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e7668b60 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3420 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9cb3438 in 0x5557e6268fc0) _discard 0x0~1000 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 _add level 1 near 0 on buffer(0x5557e9cd7560 space 0x5557e9cb3438 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 0x0~1000 = 4096 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:6336c6f2:::smithi13140796-16:head 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x34a5d 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 65536 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.668+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# hinfo_key 2019-12-07T19:33:58.668+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:6336c6f2:::smithi13140796-16:head# hinfo_key = 0 2019-12-07T19:33:58.668+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=15, attrs_read=0)) v2 -- 0x5557e9bc0000 con 0x5557e79ca800 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000131078s (0.000000111s flush + 0.000130967s kv commit) 2019-12-07T19:33:58.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.668+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e9d84580 finish 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e7668620 blob([0xa1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb32d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9cb32e8 in 0x5557e6269b20) _discard 0x0~1000 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 _add level 1 near 0 on buffer(0x5557e9cd6cf0 space 0x5557e9cb32e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 0x0~1000 = 4096 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:51ccf92f:::smithi13140796-15:head 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:51ccf92f:::smithi13140796-15:head 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 61440 2019-12-07T19:33:58.668+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# hinfo_key 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.as1_head 1#1:51ccf92f:::smithi13140796-15:head# hinfo_key = 0 2019-12-07T19:33:58.668+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.as0 21/18 ECSubReadReply(tid=34, attrs_read=0)) v2 -- 0x5557e76ca780 con 0x5557e79cac00 2019-12-07T19:33:58.668+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e9d84f20 finish 2019-12-07T19:33:58.668+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7631b00> 2019-12-07T19:33:58.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.668+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 kv_submitted 2019-12-07T19:33:58.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631b00 2019-12-07T19:33:58.668+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 finishing 2019-12-07T19:33:58.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631b00 onodes 2019-12-07T19:33:58.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631b00 done 2019-12-07T19:33:58.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.668+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631b00 [] 2019-12-07T19:33:58.668+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.668+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=13, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e9d88400 con 0x5557e79ca800 2019-12-07T19:33:58.668+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 26 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e2800 con 0x5557e79cbc00 2019-12-07T19:33:58.669+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85080 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.669+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.669+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 27 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e70a7000 con 0x5557e79cbc00 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.669+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d851e0 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.669+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85080 prio 127 cost 0 latency 0.000068 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:49, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:49 2019-12-07T19:33:58.656740+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.669+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7630900 seq 6 2019-12-07T19:33:58.669+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630900 onodes shared_blobs 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630900 prepare 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630900 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630900 io_done 2019-12-07T19:33:58.669+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85080 finish 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e9d851e0 prio 127 cost 0 latency 0.000223 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x34a5d~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x34a5d 0x34a5d~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x4a5d 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa5d 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:50, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xece and padding block with 0x132 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:50 2019-12-07T19:33:58.656793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:50 2019-12-07T19:33:58.656793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.669+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e9cc5800 seq 7 2019-12-07T19:33:58.669+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# 3 keys 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x34ece 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# 3 keys = 0 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.669+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000140019s (0.000000082s flush + 0.000139937s kv commit) 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.669+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5800 onodes 0x5557e76caf00 shared_blobs 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cb008) update 5#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:c87708e9:::smithi13140796-17:head# is 662 (485 bytes onode + 2 bytes spanning blobs + 175 bytes inline extents) 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 prepare 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5800 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 io_done 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.669+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e9d851e0 finish 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1702 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630900> 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630900 kv_submitted 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630900 2019-12-07T19:33:58.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630900 finishing 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630900 onodes 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76caf00 had 1 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630900 done 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5800 kv_submitted 2019-12-07T19:33:58.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630900 [] 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1725 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1725 < min_flush_size 524288 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x34ece~6bd to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x34ece 0x34ece~6bd to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x4ece 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xece 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.669+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=2, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e9b36400 con 0x5557e79cbc00 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x58b and padding block with 0xa75 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3558b 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.669+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000136601s (0.000000086s flush + 0.000136515s kv commit) 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc5800> 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 kv_submitted 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5800 2019-12-07T19:33:58.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 finishing 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5800 onodes 0x5557e76caf00 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5800 done 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.669+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5800 [] 2019-12-07T19:33:58.669+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.669+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9c77200 con 0x5557e79cbc00 2019-12-07T19:33:58.669+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.669+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 40 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 83892+0+0 (crc 0 0 0) 0x5557e9c79000 con 0x5557e79ca800 2019-12-07T19:33:58.669+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.669+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:58 2019-12-07T19:33:58.665940+0000 0 ObjectCleanRegions clean_offsets: [327680~18446744073709223935], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:58 2019-12-07T19:33:58.665940+0000 0 ObjectCleanRegions clean_offsets: [327680~18446744073709223935], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e9cc4900 seq 5 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.fs5_head 0x5557e75e32c0) get_onode oid 5#1:f7919c75:::smithi13140796-20:head# key 0x858000000000000001f7919c'u!smithi13140796-20!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.669+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1103 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# = 0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# 2 keys 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# 2 keys = 0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# 0x0~14000 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:f7919c75:::smithi13140796-20:head# 0x0~14000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~14000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9cc4900 10 blobs 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate 0x14000/1000,14000,0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xf6000~14000/1000,14000,0 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xf6000~14000] 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d93ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb36c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d93ea0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb36c0 sbid 0x0)) 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93ea0) get_ref 0x0~2000 Blob(0x5557e9d93ea0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb36c0 sbid 0x0)) 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9d93ea0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb36c0 sbid 0x0)) 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9cb36d8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2af0 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93f10 sbid 0x0)) 2019-12-07T19:33:58.669+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9be2af0) get_ref 0x0~2000 Blob(0x5557e9be2af0 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93f10 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9be2af0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9be2af0 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93f10 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d93f28 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d93f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d93f80 blob([0xfa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92d90 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93f80) get_ref 0x0~2000 Blob(0x5557e9d93f80 blob([0xfa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92d90 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d93f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9d93f80 blob([0xfa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92d90 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d92da8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92d20 blob([0xfc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92380 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d92d20) get_ref 0x0~2000 Blob(0x5557e9d92d20 blob([0xfc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92380 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d92d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9d92d20 blob([0xfc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92380 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d92398 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d365b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d365b0 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a000 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d365b0) get_ref 0x0~2000 Blob(0x5557e9d365b0 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a000 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d365b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9d365b0 blob([0xfe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2a000 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2a018 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2a070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2a070 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a0e0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a070) get_ref 0x0~2000 Blob(0x5557e9c2a070 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a0e0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9c2a070 blob([0x100000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2a0e0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2a0f8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2a150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2a150 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a1c0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a150) get_ref 0x0~2000 Blob(0x5557e9c2a150 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a1c0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9c2a150 blob([0x102000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2a1c0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2a1d8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2a230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2a230 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a2a0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a230) get_ref 0x0~2000 Blob(0x5557e9c2a230 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a2a0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9c2a230 blob([0x104000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2a2a0 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2a2b8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2a310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2a310 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a380 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a310) get_ref 0x0~2000 Blob(0x5557e9c2a310 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a380 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9c2a310 blob([0x106000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2a380 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2a398 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2a3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2a3f0 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a460 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a3f0) get_ref 0x0~2000 Blob(0x5557e9c2a3f0 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a460 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2a3f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9c2a3f0 blob([0x108000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2a460 sbid 0x0)) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2a478 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x14000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9bc1508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# 0x0~14000 = 0 2019-12-07T19:33:58.670+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.670+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4900 cost 128350 (11 ios * 4000 + 84350 bytes) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4900 onodes 0x5557e9bc1400 shared_blobs 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9bc1508) update 5#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9bc1508) update inline shard 193 bytes from 10 extents 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:f7919c75:::smithi13140796-20:head# is 612 (413 bytes onode + 2 bytes spanning blobs + 197 bytes inline extents) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4900 allocated 0x[f6000~14000] released 0x[] 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 freelist allocate 0xf6000~14000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 freelist _xor first_key 0x80000 last_key 0x100000 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 prepare 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9cc4900 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.670+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85340 finish 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.670+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 aio_wait 2019-12-07T19:33:58.670+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4900 2019-12-07T19:33:58.670+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 io_done 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2167 < min_flush_size 524288 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc1400 had 1 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2190 < min_flush_size 524288 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2190 < min_flush_size 524288 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3558b~88e to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3558b 0x3558b~88e to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x558b 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x58b 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe19 and padding block with 0x1e7 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x35e19 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.670+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000136304s (0.000042458s flush + 0.000093846s kv commit) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.670+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4900> 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.670+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 kv_submitted 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4900 2019-12-07T19:33:58.670+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 finishing 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4900 onodes 0x5557e9bc1400 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2a018 in 0x5557e7072000) _finish_write discard buffer(0x5557e79aa990 space 0x5557e9c2a018 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2a0f8 in 0x5557e7072000) _finish_write discard buffer(0x5557e79ab950 space 0x5557e9c2a0f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2a1d8 in 0x5557e7072000) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9c2a1d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2a2b8 in 0x5557e7072000) _finish_write discard buffer(0x5557e79aa120 space 0x5557e9c2a2b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2a398 in 0x5557e7072000) _finish_write discard buffer(0x5557e9b58c60 space 0x5557e9c2a398 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2a478 in 0x5557e7072000) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9c2a478 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb36d8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd79e0 space 0x5557e9cb36d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92398 in 0x5557e7072000) _finish_write discard buffer(0x5557e79aa360 space 0x5557e9d92398 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92da8 in 0x5557e7072000) _finish_write discard buffer(0x5557e79aac60 space 0x5557e9d92da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93f28 in 0x5557e7072000) _finish_write discard buffer(0x5557e79ab7a0 space 0x5557e9d93f28 0x0~2000 writing nocache) 2019-12-07T19:33:58.670+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=14, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e9c76600 con 0x5557e79ca800 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4900 done 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.670+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4900 [] 2019-12-07T19:33:58.670+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.670+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.670+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.671+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 66 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=35, reqid=client.4540.0:61, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 87988+0+0 (crc 0 0 0) 0x5557e9d80000 con 0x5557e79cac00 2019-12-07T19:33:58.671+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4c60 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=35, reqid=client.4540.0:61, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.671+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=35, reqid=client.4540.0:61, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.671+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=35, reqid=client.4540.0:61, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.671+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.671+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=35, reqid=client.4540.0:61, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.671+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4c60 prio 127 cost 0 latency 0.000068 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=35, reqid=client.4540.0:61, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.671+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=35, reqid=client.4540.0:61, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.671+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:61 2019-12-07T19:33:58.668524+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.671+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:61 2019-12-07T19:33:58.668524+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.671+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.671+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.671+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.671+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7121500 seq 11 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:15219530:::smithi13140796-21:head# key 0x858000000000000001152195'0!smithi13140796-21!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.671+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1104 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:head# = 0 2019-12-07T19:33:58.671+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 2 keys 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 2 keys = 0 2019-12-07T19:33:58.671+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:15219530:::smithi13140796-21:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:15219530:::smithi13140796-21:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.671+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 0x0~15000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:15219530:::smithi13140796-21:head# 0x0~15000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~15000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 11 blobs 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7f2b764de700 10 fbmap_alloc 0x5557e6282e00 allocate 0x15000/1000,15000,0 2019-12-07T19:33:58.671+0000 7f2b764de700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x10a000~15000/1000,15000,0 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x10a000~15000] 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a288c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668bd0 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a288c0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e7668bd0) get_ref 0x0~2000 Blob(0x5557e7668bd0 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a288c0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e7668bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7668bd0 blob([0x10a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7a288c0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e7a288d8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c7570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb30a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c7570 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb30a0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e75c7570) get_ref 0x0~2000 Blob(0x5557e75c7570 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb30a0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e75c7570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e75c7570 blob([0x10c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb30a0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9cb30b8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c7490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c7490 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7500 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e75c7490) get_ref 0x0~2000 Blob(0x5557e75c7490 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7500 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e75c7490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e75c7490 blob([0x10e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7500 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e75c7518 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c71f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c71f0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7d50 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e75c71f0) get_ref 0x0~2000 Blob(0x5557e75c71f0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7d50 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e75c71f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e75c71f0 blob([0x110000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7d50 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e75c7d68 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2abd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669420 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2abd0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e7669420) get_ref 0x0~2000 Blob(0x5557e7669420 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2abd0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e7669420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e7669420 blob([0x112000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2abd0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2abe8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2ac40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2acb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2ac40 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2acb0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2ac40) get_ref 0x0~2000 Blob(0x5557e9c2ac40 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2acb0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2ac40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9c2ac40 blob([0x114000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2acb0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2acc8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2ad20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ad90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2ad20 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ad90 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2ad20) get_ref 0x0~2000 Blob(0x5557e9c2ad20 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ad90 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2ad20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9c2ad20 blob([0x116000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2ad90 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2ada8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669d50 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56d90 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e7669d50) get_ref 0x0~2000 Blob(0x5557e7669d50 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56d90 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e7669d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e7669d50 blob([0x118000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b56d90 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b56da8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37c70 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37ce0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9d37c70) get_ref 0x0~2000 Blob(0x5557e9d37c70 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37ce0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9d37c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9d37c70 blob([0x11a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37ce0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9d37cf8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ae00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37d50 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ae00 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9d37d50) get_ref 0x0~2000 Blob(0x5557e9d37d50 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ae00 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9d37d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9d37d50 blob([0x11c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2ae00 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2ae18 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2ae70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2ae70 blob([0x11e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aee0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2ae70) get_ref 0x0~1000 Blob(0x5557e9c2ae70 blob([0x11e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aee0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2ae70) get_ref init 0x1000, 1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~1000: 0x0~1000 Blob(0x5557e9c2ae70 blob([0x11e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9c2aee0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2aef8 in 0x5557e6269960) _discard 0x0~1000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x15000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.extentmap(0x5557e76cb288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 0x0~15000 = 0 2019-12-07T19:33:58.671+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:15219530:::smithi13140796-21:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:15219530:::smithi13140796-21:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.671+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121500 cost 135638 (12 ios * 4000 + 87638 bytes) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121500 onodes 0x5557e76cb180 shared_blobs 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.extentmap(0x5557e76cb288) update 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore.extentmap(0x5557e76cb288) update inline shard 209 bytes from 11 extents 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:15219530:::smithi13140796-21:head# is 628 (413 bytes onode + 2 bytes spanning blobs + 213 bytes inline extents) 2019-12-07T19:33:58.671+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121500 allocated 0x[10a000~15000] released 0x[] 2019-12-07T19:33:58.671+0000 7f2b764de700 10 freelist allocate 0x10a000~15000 2019-12-07T19:33:58.671+0000 7f2b764de700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 prepare 2019-12-07T19:33:58.671+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7121500 2019-12-07T19:33:58.671+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.671+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4c60 finish 2019-12-07T19:33:58.671+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.671+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 aio_wait 2019-12-07T19:33:58.672+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121500 2019-12-07T19:33:58.672+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 io_done 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1340 < min_flush_size 524288 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb180 had 1 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1363 < min_flush_size 524288 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1363 < min_flush_size 524288 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x35e19~553 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x35e19 0x35e19~553 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x5e19 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe19 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x36c and padding block with 0xc94 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3636c 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.672+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000248172s (0.000044461s flush + 0.000203711s kv commit) 2019-12-07T19:33:58.672+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121500> 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.672+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121500 2019-12-07T19:33:58.672+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 finishing 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121500 onodes 0x5557e76cb180 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c7518 in 0x5557e6269960) _finish_write discard buffer(0x5557e79ab950 space 0x5557e75c7518 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c7d68 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e75c7d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a288d8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aaea0 space 0x5557e7a288d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56da8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aac60 space 0x5557e9b56da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2abe8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58c60 space 0x5557e9c2abe8 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2acc8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aa120 space 0x5557e9c2acc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2ada8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aa360 space 0x5557e9c2ada8 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2ae18 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9c2ae18 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2aef8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9c2aef8 0x0~1000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=35, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e76b1e00 con 0x5557e79cac00 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb30b8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aa990 space 0x5557e9cb30b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37cf8 in 0x5557e6269960) _finish_write discard buffer(0x5557e79ab7a0 space 0x5557e9d37cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 done 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.672+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121500 [] 2019-12-07T19:33:58.672+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.672+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.672+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.672+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.672+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 67 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d80800 con 0x5557e79cac00 2019-12-07T19:33:58.672+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.672+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.672+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.672+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.672+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.672+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.672+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 68 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e763f000 con 0x5557e79cac00 2019-12-07T19:33:58.672+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.672+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.672+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000073 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.673+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7046f00 seq 10 2019-12-07T19:33:58.673+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes shared_blobs 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:58.673+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000233 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3636c~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3636c 0x3636c~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x636c 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x36c 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x7dd and padding block with 0x823 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.673+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e9cc5200 seq 11 2019-12-07T19:33:58.673+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 3 keys 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 3 keys = 0 2019-12-07T19:33:58.673+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.673+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5200 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5200 onodes 0x5557e9c7c780 shared_blobs 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore.extentmap(0x5557e9c7c888) update 1#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:ec1285ab:::smithi13140796-19:head# is 681 (485 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 prepare 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5200 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 io_done 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x367dd 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.673+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c4420 finish 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000159829s (0.000000086s flush + 0.000159743s kv commit) 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1721 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7c780 had 1 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1744 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1744 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x367dd~6d0 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x367dd 0x367dd~6d0 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x67dd 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x7dd 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xead and padding block with 0x153 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.673+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:58.673+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5200 kv_submitted 2019-12-07T19:33:58.673+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:58.673+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.673+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=36, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e9b37000 con 0x5557e79cac00 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x36ead 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.673+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000484258s (0.000000055s flush + 0.000484203s kv commit) 2019-12-07T19:33:58.673+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc5200> 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.673+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 kv_submitted 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5200 2019-12-07T19:33:58.673+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 finishing 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5200 onodes 0x5557e9c7c780 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5200 done 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.673+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5200 [] 2019-12-07T19:33:58.673+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.673+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.673+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.673+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=37, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e76b1200 con 0x5557e79cac00 2019-12-07T19:33:58.673+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.673+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 28 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763c800 con 0x5557e79cbc00 2019-12-07T19:33:58.673+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c42c0 prio 127 cost 0 latency 0.000035 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.673+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.673+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.673+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c42c0 prio 127 cost 0 latency 0.000078 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.674+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b10f00 seq 6 2019-12-07T19:33:58.674+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10f00 onodes shared_blobs 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 prepare 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 io_done 2019-12-07T19:33:58.674+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c42c0 finish 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x36ead~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x36ead 0x36ead~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x6ead 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xead 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x31e and padding block with 0xce2 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3731e 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.674+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000143137s (0.000000090s flush + 0.000143047s kv commit) 2019-12-07T19:33:58.674+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.674+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.674+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10f00> 2019-12-07T19:33:58.674+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.674+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.674+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10f00 2019-12-07T19:33:58.674+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 finishing 2019-12-07T19:33:58.674+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10f00 onodes 2019-12-07T19:33:58.674+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 done 2019-12-07T19:33:58.674+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.674+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10f00 [] 2019-12-07T19:33:58.674+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.674+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.674+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=4, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e758da00 con 0x5557e79cbc00 2019-12-07T19:33:58.674+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 29 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e763d000 con 0x5557e79cbc00 2019-12-07T19:33:58.675+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.675+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000092 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.675+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 10 ==== osd_op(client.4540.0:67 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 264+0+376832 (crc 0 0 0) 0x5557e9b55340 con 0x5557e6f87800 2019-12-07T19:33:58.675+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84dc0 prio 63 cost 376832 latency 0.000016 epoch 21 osd_op(client.4540.0:67 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.675+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:67 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.675+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 11 ==== osd_op(client.4540.0:68 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 226+0+63 (crc 0 0 0) 0x5557e9b54dc0 con 0x5557e6f87800 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:67 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) queued 2019-12-07T19:33:58.675+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84000 prio 63 cost 63 latency 0.000010 epoch 21 osd_op(client.4540.0:68 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.675+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:68 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:67 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.675+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.675+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 prio 63 cost 376832 latency 0.000058 osd_op(client.4540.0:67 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.675+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:68 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b11b00 seq 7 2019-12-07T19:33:58.675+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 12 ==== osd_op(client.4540.0:69 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 226+0+0 (crc 0 0 0) 0x5557e9b54b00 con 0x5557e6f87800 2019-12-07T19:33:58.675+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d854a0 prio 63 cost 0 latency 0.000011 epoch 21 osd_op(client.4540.0:69 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.675+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 3 keys 2019-12-07T19:33:58.675+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: osd_op(client.4540.0:67 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 3 keys = 0 2019-12-07T19:33:58.675+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.675+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op: op osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11b00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11b00 onodes 0x5557e9c7c000 shared_blobs 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9c7c108) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 665 (485 bytes onode + 2 bytes spanning blobs + 178 bytes inline extents) 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 prepare 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11b00 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 io_done 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: obc NOT found in cache: 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.675+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# _ 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:head# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# _ = -2 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: no obc for soid 1:3a5a3f5b:::smithi13140796-23:head but can_create 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] create_object_context 0x5557e760c480 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] populate_obc_watchers 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: 0x5557e760c480 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x5557e628b200 snapset: 0=[]:{} 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.675+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 0x5557e70b7800 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat] ov 0'0 av 21'7 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat] 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op append 0~376832 in=376832b 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [write 0~376832 in=376832b] 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op write 0~376832 in=376832b 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] clean_regions modifiedclean_offsets: [376832~18446744073709174783], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_osd_op stat 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] stat oi has 376832 0.000000 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1705 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] make_writeable 1:3a5a3f5b:::smithi13140796-23:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] setting DIRTY flag 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] make_writeable 1:3a5a3f5b:::smithi13140796-23:head done, snapset=0=[]:{} 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7c000 had 1 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1728 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1728 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] finish_ctx 1:3a5a3f5b:::smithi13140796-23:head 0x5557e70b7800 op modify 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3731e~6c0 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3731e 0x3731e~6c0 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] set mtime to 2019-12-07T19:33:58.673753+0000 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x731e 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x31e 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x9de and padding block with 0x622 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] final snapset 0=[]:{} in 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] finish_ctx object 1:3a5a3f5b:::smithi13140796-23:head marks clean_regions clean_offsets: [376832~18446744073709174783], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x379de 2019-12-07T19:33:58.675+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 30 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763d800 con 0x5557e79cbc00 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.675+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.675+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000102398s (0.000000089s flush + 0.000102309s kv commit) 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.675+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.675+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11b00> 2019-12-07T19:33:58.675+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.675+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 kv_submitted 2019-12-07T19:33:58.675+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11b00 2019-12-07T19:33:58.675+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 finishing 2019-12-07T19:33:58.675+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11b00 onodes 0x5557e9c7c000 2019-12-07T19:33:58.675+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11b00 done 2019-12-07T19:33:58.675+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.675+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11b00 [] 2019-12-07T19:33:58.675+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.675+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000080 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=6, reqid=client.4540.0:50, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:50 2019-12-07T19:33:58.656793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.675+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e9b11800 seq 8 2019-12-07T19:33:58.675+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11800 onodes shared_blobs 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 prepare 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11800 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 io_done 2019-12-07T19:33:58.675+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=5, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9c77000 con 0x5557e79cbc00 2019-12-07T19:33:58.675+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.675+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x379de~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.675+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x379de 0x379de~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x79de 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x9de 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.675+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe4f and padding block with 0x1b1 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] execute_ctx alloc reply 0x5557e9d59b80 result 0 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] calc_trim_to_aggressive limit = 21'6 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] op order client.4540 tid 67 (first) 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] new_repop rep_tid 10 on osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] new_repop: repgather(0x5557e9cc0000 0'0 rep_tid=10 committed?=0 r=0) 2019-12-07T19:33:58.675+0000 7f2b784e2700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] issue_repop rep_tid 10 o 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:head v=21'7 tt=0'0 tid=10 reqid=client.4540.0:67 client_op=osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.675+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:33:58.675+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:head# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 376832 2019-12-07T19:33:58.676+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=21'7 tt=0'0 tid=10 reqid=client.4540.0:67 client_op=osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[0~376832]}) 2019-12-07T19:33:58.676+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:head v=21'7 tt=0'0 tid=10 reqid=client.4540.0:67 client_op=osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[0~376832]}) 2019-12-07T19:33:58.676+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=21'7 tt=0'0 tid=10 reqid=client.4540.0:67 client_op=osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[0~376832]}) 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: ExtentCache( Extents(1:3a5a3f5b:::smithi13140796-23:head)[ Extent(0~376832:7) ) 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 0 buffers are {0~376832(376832)} 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] operator(): adding buffer_update 0,376832 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] operator(): to_append: {0~376832(376832)} 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] operator(): appending 0~376832 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] encode_and_write: 1:3a5a3f5b:::smithi13140796-23:head new_size 376832 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 376832 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] operator(): marking append 0 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: ExtentCache( Extents(1:3a5a3f5b:::smithi13140796-23:head)[ Extent(0~376832:7) ) 2019-12-07T19:33:58.676+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:head={0~376832(376832)}} 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x37e4f 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=21'7 tt=0'0 tid=10 reqid=client.4540.0:67 client_op=osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[0~376832]}) 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:head=[0~376832]} 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.001214610s (0.000000066s flush + 0.001214544s kv commit) 2019-12-07T19:33:58.677+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.677+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11800> 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{0~376832(376832)} 2019-12-07T19:33:58.677+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.677+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 kv_submitted 2019-12-07T19:33:58.677+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11800 2019-12-07T19:33:58.677+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 finishing 2019-12-07T19:33:58.677+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11800 onodes 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.677+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11800 done 2019-12-07T19:33:58.677+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.677+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11800 [] 2019-12-07T19:33:58.677+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.677+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.677+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:67, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e763e000 con 0x5557e79cac00 2019-12-07T19:33:58.677+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 69 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e5000 con 0x5557e79cac00 2019-12-07T19:33:58.677+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.677+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.677+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:67, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e763f800 con 0x5557e7519800 2019-12-07T19:33:58.677+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:67, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e763e800 con 0x5557e79ca800 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.677+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=10, reqid=client.4540.0:67, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e763c000 con 0x5557e79cbc00 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.677+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:67, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 -- 0x5557e9ba1800 con 0x5557e79ca000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 earliest_dup_version = 0 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 trim 21'7 (0'0) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:67 2019-12-07T19:33:58.673753+0000 0 ObjectCleanRegions clean_offsets: [376832~18446744073709174783], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] append_log log((0'0,21'6], crt=21'6) [21'7 (0'0) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:67 2019-12-07T19:33:58.673753+0000 0 ObjectCleanRegions clean_offsets: [376832~18446744073709174783], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 lua=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] add_log_entry 21'7 (0'0) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:67 2019-12-07T19:33:58.673753+0000 0 ObjectCleanRegions clean_offsets: [376832~18446744073709174783], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 lua=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] append_log approx pg log length = 7 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 lua=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.677+0000 7f2b784e2700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7a93b00 seq 13 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3a5a3f5b:::smithi13140796-23:head# key 0x8080000000000000013a5a3f'[!smithi13140796-23!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.677+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.677+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1105 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# = 0 2019-12-07T19:33:58.677+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2 keys = 0 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7a93800 seq 12 2019-12-07T19:33:58.677+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.677+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.677+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~17000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~17000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93800 onodes shared_blobs 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a93b00 12 blobs 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 prepare 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93800 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 io_done 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4420 finish 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 fbmap_alloc 0x5557e6282e00 allocate 0x17000/1000,17000,0 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x11f000~17000/1000,17000,0 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x11f000~17000] 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b704d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b703f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b704d0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b703f0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b704d0) get_ref 0x0~2000 Blob(0x5557e9b704d0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b703f0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b704d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9b704d0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b703f0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b70408 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b708c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b708c0 blob([0x121000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70930 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b708c0) get_ref 0x0~2000 Blob(0x5557e9b708c0 blob([0x121000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70930 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b708c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9b708c0 blob([0x121000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70930 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b70948 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71500 blob([0x123000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71340 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71500) get_ref 0x0~2000 Blob(0x5557e9b71500 blob([0x123000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71340 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9b71500 blob([0x123000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b71340 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b71358 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b712d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71490 blob([0x125000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b712d0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71490) get_ref 0x0~2000 Blob(0x5557e9b71490 blob([0x125000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b712d0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9b71490 blob([0x125000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b712d0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b712e8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71420 blob([0x127000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71260 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71420) get_ref 0x0~2000 Blob(0x5557e9b71420 blob([0x127000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71260 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9b71420 blob([0x127000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b71260 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b71278 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b711f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 41 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e3800 con 0x5557e79ca800 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b711f0 blob([0x129000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71030 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b711f0) get_ref 0x0~2000 Blob(0x5557e9b711f0 blob([0x129000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71030 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b711f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c51e0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9b711f0 blob([0x129000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b71030 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b71048 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71180 blob([0x12b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70fc0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71180) get_ref 0x0~2000 Blob(0x5557e9b71180 blob([0x12b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70fc0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9b71180 blob([0x12b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70fc0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b70fd8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71110 blob([0x12d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70f50 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71110) get_ref 0x0~2000 Blob(0x5557e9b71110 blob([0x12d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70f50 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9b71110 blob([0x12d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70f50 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b70f68 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b710a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b710a0 blob([0x12f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70ee0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b710a0) get_ref 0x0~2000 Blob(0x5557e9b710a0 blob([0x12f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70ee0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b710a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9b710a0 blob([0x12f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70ee0 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b70ef8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b70e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b70e70 blob([0x131000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70e00 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b70e70) get_ref 0x0~2000 Blob(0x5557e9b70e70 blob([0x131000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70e00 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b70e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9b70e70 blob([0x131000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70e00 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b70e18 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b70d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b70d90 blob([0x133000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70d20 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b70d90) get_ref 0x0~2000 Blob(0x5557e9b70d90 blob([0x133000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70d20 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b70d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9b70d90 blob([0x133000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70d20 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b70d38 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b70cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b70cb0 blob([0x135000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70c40 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b70cb0) get_ref 0x0~1000 Blob(0x5557e9b70cb0 blob([0x135000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70c40 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b70cb0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~1000: 0x0~1000 Blob(0x5557e9b70cb0 blob([0x135000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9b70c40 sbid 0x0)) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b70c58 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x17000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e9b94388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~17000 = 0 2019-12-07T19:33:58.677+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.677+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93b00 cost 191830 (24 ios * 4000 + 95830 bytes) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93b00 onodes 0x5557e9b94280 shared_blobs 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e9b94388) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e9b94388) update inline shard 228 bytes from 12 extents 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 647 (413 bytes onode + 2 bytes spanning blobs + 232 bytes inline extents) 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93b00 allocated 0x[11f000~17000] released 0x[] 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 freelist allocate 0x11f000~17000 2019-12-07T19:33:58.677+0000 7f2b784e2700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 prepare 2019-12-07T19:33:58.677+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a93b00 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c51e0 prio 127 cost 0 latency 0.000190 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:58 2019-12-07T19:33:58.665940+0000 0 ObjectCleanRegions clean_offsets: [327680~18446744073709223935], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.677+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e9cc5500 seq 6 2019-12-07T19:33:58.677+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5500 onodes shared_blobs 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 prepare 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5500 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 io_done 2019-12-07T19:33:58.677+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c51e0 finish 2019-12-07T19:33:58.677+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 70 ==== MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e76ca780 con 0x5557e79cac00 2019-12-07T19:33:58.677+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c46e0 prio 127 cost 0 latency 0.000034 epoch 21 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.677+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 433 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2239 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2262 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2262 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x37e4f~8d6 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x37e4f 0x37e4f~8d6 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x7e4f 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe4f 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x725 and padding block with 0x8db 2019-12-07T19:33:58.677+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x38725 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.677+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=6, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e76b1200 con 0x5557e79cbc00 2019-12-07T19:33:58.677+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.677+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c46e0 prio 127 cost 0 latency 0.000238 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.677+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.677+0000 7f2b72cd7700 25 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.677+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 0x0~1000 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x13000 (77824) 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000205959s (0.000000094s flush + 0.000205865s kv commit) 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7621dc0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2540 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a93800,0x5557e9cc5500> 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e7621dc0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2540 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 kv_submitted 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93800 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 finishing 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93800 onodes 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93800 done 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93800 [] 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 kv_submitted 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5500 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 finishing 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5500 onodes 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5500 done 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5500 [] 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.678+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 31 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763e000 con 0x5557e79cbc00 2019-12-07T19:33:58.678+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5760 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.678+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.678+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 32 ==== MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9bc0000 con 0x5557e79cbc00 2019-12-07T19:33:58.678+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.678+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 lua=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] op_applied version 21'7 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c5760 prio 127 cost 0 latency 0.000130 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] eval_repop repgather(0x5557e9cc0000 21'7 rep_tid=10 committed?=0 r=0) 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 finish 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.678+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.678+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:68 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.678+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84000 prio 63 cost 63 latency 0.003337 osd_op(client.4540.0:68 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: osd_op(client.4540.0:68 1.cs0 1.dafc5a5c (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7693b00 seq 8 2019-12-07T19:33:58.678+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.678+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op: op osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.678+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693b00 onodes shared_blobs 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 prepare 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693b00 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 io_done 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c5760 finish 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: found obc in cache: 0x5557e760c480 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: 0x5557e760c480 1:3a5a3f5b:::smithi13140796-23:head rwstate(excl n=1 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e628b200 snapset: 0=[]:{} 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000204 MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.678+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.678+0000 7f2b74cdb700 25 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.678+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.678+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~1000 2019-12-07T19:33:58.678+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84000 finish 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x12000 (73728) 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.678+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c6e00 blob([0xd1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2930 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e75c6e00 blob([0xd1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2930 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.678+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.678+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d854a0 prio 63 cost 0 latency 0.003445 osd_op(client.4540.0:69 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.678+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 aio_wait 2019-12-07T19:33:58.678+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93b00 2019-12-07T19:33:58.678+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 io_done 2019-12-07T19:33:58.678+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: osd_op(client.4540.0:69 1.cs0 1.dafc5a5c (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e7621dc0 blob([0xe3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2540 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9be2558 in 0x5557e6269880) _discard 0x0~1000 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9b590e0 space 0x5557e9be2558 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.678+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op: op osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.678+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 0x0~1000 = 4096 2019-12-07T19:33:58.678+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.678+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.678+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:33:58.678+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:33:58.678+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.678+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: found obc in cache: 0x5557e760c480 2019-12-07T19:33:58.678+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 77824 2019-12-07T19:33:58.678+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:33:58.678+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:33:58.678+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] get_object_context: 0x5557e760c480 1:3a5a3f5b:::smithi13140796-23:head rwstate(excl n=1 w=1) oi: 1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e628b200 snapset: 0=[]:{} 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7f2b72cd7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=39, attrs_read=0)) v2 -- 0x5557e9c7ca00 con 0x5557e79cac00 2019-12-07T19:33:58.678+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.678+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c46e0 finish 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.678+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.678+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2473 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.678+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d854a0 finish 2019-12-07T19:33:58.678+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b94280 had 1 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2496 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2496 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=38, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e76b1e00 con 0x5557e79cac00 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x38725~9c0 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x38725 0x38725~9c0 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.678+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=16, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e76b0000 con 0x5557e79ca800 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x8725 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x725 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.678+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe5 and padding block with 0xf1b 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e75c6e00 blob([0xd1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2930 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9be2948 in 0x5557e6269c00) _discard 0x0~1000 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 _add level 1 near 0 on buffer(0x5557e9cd6510 space 0x5557e9be2948 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~1000 = 4096 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:db85c185:::smithi13140796-18:head 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:db85c185:::smithi13140796-18:head 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 73728 2019-12-07T19:33:58.678+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# hinfo_key = 0 2019-12-07T19:33:58.678+0000 7f2b74cdb700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 -- 0x5557e795c500 con 0x5557e79cbc00 2019-12-07T19:33:58.678+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c4f20 finish 2019-12-07T19:33:58.678+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x390e5 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.678+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000263409s (0.000050901s flush + 0.000212508s kv commit) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.678+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693b00,0x5557e7a93b00> 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 kv_submitted 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693b00 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 finishing 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693b00 onodes 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693b00 done 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693b00 [] 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 kv_submitted 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93b00 2019-12-07T19:33:58.678+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 33 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e76b1200 con 0x5557e79cbc00 2019-12-07T19:33:58.678+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 finishing 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93b00 onodes 0x5557e9b94280 2019-12-07T19:33:58.678+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e76958c0 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70408 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd7710 space 0x5557e9b70408 0x0~2000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=8, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9d89400 con 0x5557e79cbc00 2019-12-07T19:33:58.678+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70948 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd77a0 space 0x5557e9b70948 0x0~2000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70c58 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd6bd0 space 0x5557e9b70c58 0x0~1000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70d38 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd73b0 space 0x5557e9b70d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70e18 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9b70e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70ef8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd7290 space 0x5557e9b70ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70f68 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd7200 space 0x5557e9b70f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70fd8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd7170 space 0x5557e9b70fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b71048 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9b71048 0x0~2000 writing nocache) 2019-12-07T19:33:58.678+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b71278 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd6000 space 0x5557e9b71278 0x0~2000 writing nocache) 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b712e8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd79e0 space 0x5557e9b712e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b71358 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd7830 space 0x5557e9b71358 0x0~2000 writing nocache) 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93b00 done 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.679+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93b00 [] 2019-12-07T19:33:58.679+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.679+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e76958c0 prio 196 cost 0 latency 0.000051 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.679+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.679+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e76958c0 finish 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 42 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9ba1800 con 0x5557e79ca800 2019-12-07T19:33:58.679+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695340 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.679+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.679+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.679+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 15 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9d89600 con 0x5557e79ca000 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e7695340 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.679+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d851e0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.679+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:59 2019-12-07T19:33:58.665983+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.679+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d851e0 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:59 2019-12-07T19:33:58.665983+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.679+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.679+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d851e0 finish 2019-12-07T19:33:58.679+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.679+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7a92600 seq 7 2019-12-07T19:33:58.679+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# 3 keys 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# 3 keys = 0 2019-12-07T19:33:58.679+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.679+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92600 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92600 onodes 0x5557e9bc1400 shared_blobs 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9bc1508) update 5#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:f7919c75:::smithi13140796-20:head# is 684 (485 bytes onode + 2 bytes spanning blobs + 197 bytes inline extents) 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 prepare 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92600 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 io_done 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.679+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e7695340 finish 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1724 < min_flush_size 524288 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc1400 had 1 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1747 < min_flush_size 524288 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1747 < min_flush_size 524288 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x390e5~6d3 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x390e5 0x390e5~6d3 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x90e5 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe5 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x7b8 and padding block with 0x848 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x397b8 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.679+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000135757s (0.000000074s flush + 0.000135683s kv commit) 2019-12-07T19:33:58.679+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92600> 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.679+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 kv_submitted 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92600 2019-12-07T19:33:58.679+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 finishing 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92600 onodes 0x5557e9bc1400 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92600 done 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.679+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92600 [] 2019-12-07T19:33:58.679+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.679+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.679+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=17, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e76b1000 con 0x5557e79ca800 2019-12-07T19:33:58.679+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 43 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9d89400 con 0x5557e79ca800 2019-12-07T19:33:58.680+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 196 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.680+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.680+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 196 cost 0 latency 0.000059 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.680+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.680+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.680+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.683+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 44 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763e800 con 0x5557e79ca800 2019-12-07T19:33:58.683+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.683+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:59 2019-12-07T19:33:58.665983+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.683+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e9cc5b00 seq 8 2019-12-07T19:33:58.683+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5b00 onodes shared_blobs 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5b00 prepare 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5b00 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5b00 io_done 2019-12-07T19:33:58.683+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9d85600 finish 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x397b8~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x397b8 0x397b8~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x97b8 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x7b8 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xc29 and padding block with 0x3d7 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x39c29 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.683+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000121989s (0.000000086s flush + 0.000121903s kv commit) 2019-12-07T19:33:58.683+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.683+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc5b00> 2019-12-07T19:33:58.683+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.683+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.683+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5b00 kv_submitted 2019-12-07T19:33:58.683+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5b00 2019-12-07T19:33:58.683+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5b00 finishing 2019-12-07T19:33:58.683+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5b00 onodes 2019-12-07T19:33:58.683+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5b00 done 2019-12-07T19:33:58.683+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.683+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5b00 [] 2019-12-07T19:33:58.683+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.683+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.683+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.683+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=18, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e9b36000 con 0x5557e79ca800 2019-12-07T19:33:58.683+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.683+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 22 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:70, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 100276+0+0 (unknown 0 0 0) 0x5557e763f800 con 0x5557e7519800 2019-12-07T19:33:58.683+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000038 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:70, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.683+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:70, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:70, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:70, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000349 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:70, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:70, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:70 2019-12-07T19:33:58.678831+0000 0 ObjectCleanRegions clean_offsets: [393216~18446744073709158399], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:70 2019-12-07T19:33:58.678831+0000 0 ObjectCleanRegions clean_offsets: [393216~18446744073709158399], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7630300 seq 9 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.5s1_head 0x5557e6281c20) get_onode oid 1#1:a8debf31:::smithi13140796-24:head# key 0x818000000000000001a8debf'1!smithi13140796-24!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.684+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1106 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# = 0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 2 keys 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 2 keys = 0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 0x0~18000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:a8debf31:::smithi13140796-24:head# 0x0~18000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~18000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7630300 12 blobs 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 fbmap_alloc 0x5557e6282e00 allocate 0x18000/1000,18000,0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x136000~18000/1000,18000,0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x136000~18000] 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be27e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be27e0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70700 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9be27e0) get_ref 0x0~2000 Blob(0x5557e9be27e0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70700 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9be27e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9be27e0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70700 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b70718 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76200e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76200e0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3030 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e76200e0) get_ref 0x0~2000 Blob(0x5557e76200e0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3030 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e76200e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e76200e0 blob([0x138000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3030 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9cb3048 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76691f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76691f0 blob([0x13a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71ab0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e76691f0) get_ref 0x0~2000 Blob(0x5557e76691f0 blob([0x13a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71ab0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e76691f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e76691f0 blob([0x13a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b71ab0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b71ac8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b719d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71a40 blob([0x13c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b719d0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b71a40) get_ref 0x0~2000 Blob(0x5557e9b71a40 blob([0x13c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b719d0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b71a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9b71a40 blob([0x13c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b719d0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b719e8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b718f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71960 blob([0x13e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b718f0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b71960) get_ref 0x0~2000 Blob(0x5557e9b71960 blob([0x13e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b718f0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b71960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9b71960 blob([0x13e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b718f0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b71908 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71880 blob([0x140000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71810 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b71880) get_ref 0x0~2000 Blob(0x5557e9b71880 blob([0x140000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71810 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b71880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9b71880 blob([0x140000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b71810 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b71828 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b717a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b709a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b717a0 blob([0x142000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b709a0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b717a0) get_ref 0x0~2000 Blob(0x5557e9b717a0 blob([0x142000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b709a0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b717a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9b717a0 blob([0x142000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b709a0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b709b8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b713b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b702a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b713b0 blob([0x144000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b702a0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b713b0) get_ref 0x0~2000 Blob(0x5557e9b713b0 blob([0x144000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b702a0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b713b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9b713b0 blob([0x144000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b702a0 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b702b8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7a000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7a000 blob([0x146000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a070 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b7a000) get_ref 0x0~2000 Blob(0x5557e9b7a000 blob([0x146000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a070 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b7a000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9b7a000 blob([0x146000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a070 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b7a088 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7a0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7a0e0 blob([0x148000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a150 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b7a0e0) get_ref 0x0~2000 Blob(0x5557e9b7a0e0 blob([0x148000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a150 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b7a0e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9b7a0e0 blob([0x148000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a150 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b7a168 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7a1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7a1c0 blob([0x14a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a230 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b7a1c0) get_ref 0x0~2000 Blob(0x5557e9b7a1c0 blob([0x14a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a230 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b7a1c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9b7a1c0 blob([0x14a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a230 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b7a248 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7a2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7a2a0 blob([0x14c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a310 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b7a2a0) get_ref 0x0~2000 Blob(0x5557e9b7a2a0 blob([0x14c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a310 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9b7a2a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9b7a2a0 blob([0x14c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a310 sbid 0x0)) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b7a328 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x18000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.extentmap(0x5557e9c7cd88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 0x0~18000 = 0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630300 cost 151926 (13 ios * 4000 + 99926 bytes) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630300 onodes 0x5557e9c7cc80 shared_blobs 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.extentmap(0x5557e9c7cd88) update 1#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore.extentmap(0x5557e9c7cd88) update inline shard 231 bytes from 12 extents 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:a8debf31:::smithi13140796-24:head# is 650 (413 bytes onode + 2 bytes spanning blobs + 235 bytes inline extents) 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630300 allocated 0x[136000~18000] released 0x[] 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 freelist allocate 0x136000~18000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 prepare 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7630300 2019-12-07T19:33:58.684+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 23 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e765e600 con 0x5557e7519800 2019-12-07T19:33:58.684+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c54a0 prio 196 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.684+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.684+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.684+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.684+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.684+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.684+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c54a0 prio 196 cost 0 latency 0.000043 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.684+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.684+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c54a0 finish 2019-12-07T19:33:58.684+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.684+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4000 finish 2019-12-07T19:33:58.684+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.684+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 aio_wait 2019-12-07T19:33:58.684+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630300 2019-12-07T19:33:58.684+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 io_done 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.684+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1362 < min_flush_size 524288 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7cc80 had 1 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.684+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1385 < min_flush_size 524288 2019-12-07T19:33:58.684+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1385 < min_flush_size 524288 2019-12-07T19:33:58.684+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.684+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x39c29~569 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.684+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x39c29 0x39c29~569 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x9c29 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xc29 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.684+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x192 and padding block with 0xe6e 2019-12-07T19:33:58.685+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3a192 2019-12-07T19:33:58.685+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.685+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.685+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.685+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.685+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000146181s (0.000042143s flush + 0.000104038s kv commit) 2019-12-07T19:33:58.685+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630300> 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.685+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 kv_submitted 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630300 2019-12-07T19:33:58.685+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 finishing 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630300 onodes 0x5557e9c7cc80 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b702b8 in 0x5557e6269180) _finish_write discard buffer(0x5557e79aa360 space 0x5557e9b702b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70718 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9b70718 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b709b8 in 0x5557e6269180) _finish_write discard buffer(0x5557e79aa120 space 0x5557e9b709b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b71828 in 0x5557e6269180) _finish_write discard buffer(0x5557e79aac60 space 0x5557e9b71828 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b71908 in 0x5557e6269180) _finish_write discard buffer(0x5557e79aaea0 space 0x5557e9b71908 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b719e8 in 0x5557e6269180) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9b719e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b71ac8 in 0x5557e6269180) _finish_write discard buffer(0x5557e79ab950 space 0x5557e9b71ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7a088 in 0x5557e6269180) _finish_write discard buffer(0x5557e79aa990 space 0x5557e9b7a088 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7a168 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd7710 space 0x5557e9b7a168 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7a248 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd77a0 space 0x5557e9b7a248 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7a328 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd6bd0 space 0x5557e9b7a328 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb3048 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9cb3048 0x0~2000 writing nocache) 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630300 done 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.685+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630300 [] 2019-12-07T19:33:58.685+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.685+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.685+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=5, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e9ef2000 con 0x5557e7519800 2019-12-07T19:33:58.685+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.685+0000 7f2b95fb3700 0 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 msgr2=0x5557e7527600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:58.685+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 msgr2=0x5557e7527600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 90 2019-12-07T19:33:58.685+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 msgr2=0x5557e7527600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.685+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=READY pgs=17 cs=2 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.685+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=READY pgs=17 cs=2 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.687+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 45 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 108468+0+0 (crc 0 0 0) 0x5557e763c000 con 0x5557e79ca800 2019-12-07T19:33:58.687+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.687+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:76 2019-12-07T19:33:58.684279+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:76 2019-12-07T19:33:58.684279+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7a92c00 seq 9 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.fs5_head 0x5557e75e32c0) get_onode oid 5#1:f9bf9ba5:::smithi13140796-26:head# key 0x858000000000000001f9bf9ba521736d'ithi13140796-26!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.687+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1107 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# = 0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 2 keys 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 2 keys = 0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1a000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92c00 13 blobs 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1a000/1000,1a000,0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x14e000~1a000/1000,1a000,0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x14e000~1a000] 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2b880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d931f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2b880 blob([0x14e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d931f0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2b880) get_ref 0x0~2000 Blob(0x5557e9c2b880 blob([0x14e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d931f0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2b880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9c2b880 blob([0x14e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d931f0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9d93208 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb37a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb37a0 blob([0x150000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70850 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9cb37a0) get_ref 0x0~2000 Blob(0x5557e9cb37a0 blob([0x150000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70850 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9cb37a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9cb37a0 blob([0x150000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70850 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b70868 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aa10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92e00 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aa10 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d92e00) get_ref 0x0~2000 Blob(0x5557e9d92e00 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aa10 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9d92e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9d92e00 blob([0x152000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2aa10 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2aa28 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2aa80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2aa80 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aaf0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2aa80) get_ref 0x0~2000 Blob(0x5557e9c2aa80 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2aaf0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2aa80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9c2aa80 blob([0x154000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2aaf0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2ab08 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2ab60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7abd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2ab60 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7abd0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2ab60) get_ref 0x0~2000 Blob(0x5557e9c2ab60 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7abd0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c2ab60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9c2ab60 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7abd0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7abe8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7ac40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7acb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7ac40 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7acb0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7ac40) get_ref 0x0~2000 Blob(0x5557e9b7ac40 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7acb0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7ac40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9b7ac40 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7acb0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7acc8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7ad20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7ad90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7ad20 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7ad90 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7ad20) get_ref 0x0~2000 Blob(0x5557e9b7ad20 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7ad90 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7ad20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9b7ad20 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7ad90 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7ada8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7ae00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7ae70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7ae00 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7ae70 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7ae00) get_ref 0x0~2000 Blob(0x5557e9b7ae00 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7ae70 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7ae00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9b7ae00 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7ae70 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7ae88 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7aee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7af50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7aee0 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7af50 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7aee0) get_ref 0x0~2000 Blob(0x5557e9b7aee0 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7af50 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7aee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9b7aee0 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7af50 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7af68 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7afc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7afc0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b030 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7afc0) get_ref 0x0~2000 Blob(0x5557e9b7afc0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b030 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7afc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9b7afc0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b030 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7b048 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7b0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7b0a0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b110 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b0a0) get_ref 0x0~2000 Blob(0x5557e9b7b0a0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b110 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b0a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9b7b0a0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b110 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7b128 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7b180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7b180 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b1f0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b180) get_ref 0x0~2000 Blob(0x5557e9b7b180 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b1f0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9b7b180 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b1f0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7b208 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7b260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7b260 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b2d0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b260) get_ref 0x0~2000 Blob(0x5557e9b7b260 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b2d0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9b7b260 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b2d0 sbid 0x0)) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7b2e8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1a000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b94d88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 = 0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 164118 (14 ios * 4000 + 108118 bytes) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes 0x5557e9b94c80 shared_blobs 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b94d88) update 5#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b94d88) update inline shard 250 bytes from 13 extents 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:f9bf9ba5:::smithi13140796-26:head# is 669 (413 bytes onode + 2 bytes spanning blobs + 254 bytes inline extents) 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[14e000~1a000] released 0x[] 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 freelist allocate 0x14e000~1a000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:58.687+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a92c00 2019-12-07T19:33:58.688+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.688+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85760 finish 2019-12-07T19:33:58.688+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.688+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 aio_wait 2019-12-07T19:33:58.688+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:58.688+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1381 < min_flush_size 524288 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b94c80 had 1 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1404 < min_flush_size 524288 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1404 < min_flush_size 524288 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3a192~57c to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3a192 0x3a192~57c to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xa192 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x192 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x70e and padding block with 0x8f2 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3a70e 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.688+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000122697s (0.000038297s flush + 0.000084400s kv commit) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.688+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92c00> 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.688+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:58.688+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 0x5557e9b94c80 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70868 in 0x5557e7072000) _finish_write discard buffer(0x5557e79aa750 space 0x5557e9b70868 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7abe8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd7290 space 0x5557e9b7abe8 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7acc8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd7200 space 0x5557e9b7acc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7ada8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd7170 space 0x5557e9b7ada8 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7ae88 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd7b90 space 0x5557e9b7ae88 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7af68 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd7b00 space 0x5557e9b7af68 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7b048 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd7a70 space 0x5557e9b7b048 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7b128 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9b7b128 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=20, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e76b1c00 con 0x5557e79ca800 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7b208 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9b7b208 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7b2e8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e9b7b2e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2aa28 in 0x5557e7072000) _finish_write discard buffer(0x5557e79ab170 space 0x5557e9c2aa28 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2ab08 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9c2ab08 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93208 in 0x5557e7072000) _finish_write discard buffer(0x5557e79ab680 space 0x5557e9d93208 0x0~2000 writing nocache) 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.688+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:58.688+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.688+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.688+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.688+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 71 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e76b1c00 con 0x5557e79cac00 2019-12-07T19:33:58.688+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d858c0 prio 196 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.688+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.688+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.688+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.688+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.688+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.688+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d858c0 prio 196 cost 0 latency 0.000061 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.688+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.688+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=21'7 tt=0'0 tid=10 reqid=client.4540.0:67 client_op=osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[0~376832]}) 2019-12-07T19:33:58.688+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'6 crt=21'6 lcod 21'5 mlcod 21'5 active+clean] repop_all_committed: repop tid 10 all committed 2019-12-07T19:33:58.688+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] eval_repop repgather(0x5557e9cc0000 21'7 rep_tid=10 committed?=1 r=0) 2019-12-07T19:33:58.688+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] commit: repgather(0x5557e9cc0000 21'7 rep_tid=10 committed?=1 r=0) 2019-12-07T19:33:58.688+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] log_op_stats osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 376832 outb 0 lat 0.013770 2019-12-07T19:33:58.688+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] sending reply on osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x5557e9d59b80 2019-12-07T19:33:58.688+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(67 smithi13140796-23 [append 0~376832,stat] v21'7 uv7 ondisk = 0) v8 -- 0x5557e9d59b80 con 0x5557e6f87800 2019-12-07T19:33:58.688+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.688+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] publish_stats_to_osd 21:22 2019-12-07T19:33:58.688+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] removing repgather(0x5557e9cc0000 21'7 rep_tid=10 committed?=1 r=0) 2019-12-07T19:33:58.688+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] q front is repgather(0x5557e9cc0000 21'7 rep_tid=10 committed?=1 r=0) 2019-12-07T19:33:58.688+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] remove_repop repgather(0x5557e9cc0000 21'7 rep_tid=10 committed?=1 r=0) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] requeue_op 0x5557e9d854a0 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] requeue_op 0x5557e9d84000 2019-12-07T19:33:58.689+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) shuffled w/ OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=21'7 tt=0'0 tid=10 reqid=client.4540.0:67 client_op=osd_op(client.4540.0:67 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [append 0~376832 in=376832b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'6 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[0~376832]}) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] try_finish_rmw: ExtentCache( Extents(1:3a5a3f5b:::smithi13140796-23:head)[ Extent(0~376832:7) ) 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=11 reqid=client.4540.0:67 roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=11 reqid=client.4540.0:67 roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.689+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:67, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e9d81000 con 0x5557e79cac00 2019-12-07T19:33:58.689+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 72 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 104372+0+0 (crc 0 0 0) 0x5557e79e3000 con 0x5557e79cac00 2019-12-07T19:33:58.689+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=11, reqid=client.4540.0:67, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e9d81800 con 0x5557e7519800 2019-12-07T19:33:58.689+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85a20 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.689+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.689+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:67, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e9d82000 con 0x5557e79ca800 2019-12-07T19:33:58.689+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:67, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e9d82800 con 0x5557e79cbc00 2019-12-07T19:33:58.689+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=STANDBY pgs=17 cs=3 l=0 rx=0 tx=0).connect 2019-12-07T19:33:58.689+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:67, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e9d83000 con 0x5557e79ca000 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'6 lcod 21'6 mlcod 21'6 active+clean] append_log log((0'0,21'7], crt=21'6) [] 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] rollforward: entry=21'7 (0'0) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:67 2019-12-07T19:33:58.673753+0000 0 ObjectCleanRegions clean_offsets: [376832~18446744073709174783], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log approx pg log length = 7 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.689+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.689+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.689+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.689+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9d5ac00 seq 14 2019-12-07T19:33:58.689+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.689+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.689+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.689+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.689+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.689+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5ac00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5ac00 onodes shared_blobs 2019-12-07T19:33:58.689+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5ac00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.689+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 prepare 2019-12-07T19:33:58.689+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5ac00 2019-12-07T19:33:58.689+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 io_done 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d858c0 finish 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84000 prio 63 cost 63 latency 0.014278 osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op: op osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: found obc in cache: 0x5557e760c480 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: 0x5557e760c480 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e628b200 snapset: 0=[]:{} 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3a70e~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3a70e 0x3a70e~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xa70e 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x70e 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85a20 prio 127 cost 0 latency 0.000380 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xb7f and padding block with 0x481 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.689+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(21'7 client.4540.0:67 dirty|data_digest s 376832 uv 7 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx 0x5557e9cee300 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3ab7f 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.689+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.689+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] ov 21'7 av 21'8 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000104250s (0.000000075s flush + 0.000104175s kv commit) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e9cc4000 seq 13 2019-12-07T19:33:58.689+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.689+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_osd_op setxattr _header (56) in=63b 2019-12-07T19:33:58.689+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5ac00> 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.7s1_head 0x5557e76a92c0) get_onode oid 1#1:ee170468:::smithi13140796-25:head# key 0x818000000000000001ee1704'h!smithi13140796-25!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.689+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.689+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 kv_submitted 2019-12-07T19:33:58.689+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5ac00 2019-12-07T19:33:58.689+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1108 2019-12-07T19:33:58.689+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 finishing 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# = 0 2019-12-07T19:33:58.689+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5ac00 onodes 2019-12-07T19:33:58.689+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5ac00 done 2019-12-07T19:33:58.689+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 2 keys 2019-12-07T19:33:58.689+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] make_writeable 1:3a5a3f5b:::smithi13140796-23:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.689+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5ac00 [] 2019-12-07T19:33:58.689+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 2 keys = 0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.689+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] make_writeable 1:3a5a3f5b:::smithi13140796-23:head done, snapset=0=[]:{} 2019-12-07T19:33:58.689+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~19000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] finish_ctx 1:3a5a3f5b:::smithi13140796-23:head 0x5557e9cee300 op modify 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] set mtime to 2019-12-07T19:33:58.673986+0000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9cc4000 13 blobs 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] final snapset 0=[]:{} in 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] finish_ctx object 1:3a5a3f5b:::smithi13140796-23:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx alloc reply 0x5557e9d59680 result 0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] calc_trim_to_aggressive limit = 21'7 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op order client.4540 tid 68 last was 67 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate 0x19000/1000,19000,0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x168000~19000/1000,19000,0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x168000~19000] 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] new_repop rep_tid 12 on osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b707e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b707e0) get_ref 0x0~2000 Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b707e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] new_repop: repgather(0x5557e9cc0000 0'0 rep_tid=12 committed?=0 r=0) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2b908 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b744da700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] issue_repop rep_tid 12 o 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7b8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7b8f0 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b960 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b8f0) get_ref 0x0~2000 Blob(0x5557e9b7b8f0 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7b960 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b8f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9b7b8f0 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b960 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7b978 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7ba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7ba40 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bab0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7ba40) get_ref 0x0~2000 Blob(0x5557e9b7ba40 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bab0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7ba40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9b7ba40 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bab0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bac8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7bb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7bb20 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bb90 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bb20) get_ref 0x0~2000 Blob(0x5557e9b7bb20 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bb90 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bb20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] submit_transaction: op Op(1:3a5a3f5b:::smithi13140796-23:head v=21'8 tt=0'0 tid=12 reqid=client.4540.0:68 client_op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9b7bb20 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bb90 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bba8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7bc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7bc00 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bc70 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bc00) get_ref 0x0~2000 Blob(0x5557e9b7bc00 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bc70 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bc00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9b7bc00 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bc70 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.689+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bc88 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_hash_info: found on disk, size 94208 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7bce0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bd50 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bce0) get_ref 0x0~2000 Blob(0x5557e9b7bce0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bd50 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9b7bce0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bd50 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bd68 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7bdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7be30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7bdc0 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7be30 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bdc0) get_ref 0x0~2000 Blob(0x5557e9b7bdc0 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7be30 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bdc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9b7bdc0 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7be30 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7be48 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): 1:3a5a3f5b:::smithi13140796-23:head projected size 376832 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7bea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7bea0 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bf10 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bea0) get_ref 0x0~2000 Blob(0x5557e9b7bea0 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7bf10 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9b7bea0 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bf10 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bf28 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7bf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7bf80 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36000 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] start_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=21'8 tt=0'0 tid=12 reqid=client.4540.0:68 client_op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bf80) get_ref 0x0~2000 Blob(0x5557e9b7bf80 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36000 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7bf80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9b7bf80 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36000 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36018 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c360e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36070 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c360e0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_state_to_reads: Op(1:3a5a3f5b:::smithi13140796-23:head v=21'8 tt=0'0 tid=12 reqid=client.4540.0:68 client_op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36070) get_ref 0x0~2000 Blob(0x5557e9c36070 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c360e0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9c36070 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c360e0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c360f8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c361c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36150 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c361c0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=21'8 tt=0'0 tid=12 reqid=client.4540.0:68 client_op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36150) get_ref 0x0~2000 Blob(0x5557e9c36150 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c361c0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9c36150 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c361c0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c361d8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c362a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36230 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c362a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36230) get_ref 0x0~2000 Blob(0x5557e9c36230 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c362a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9c36230 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c362a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c362b8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36310 blob([0x180000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36380 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36310) get_ref 0x0~1000 Blob(0x5557e9c36310 blob([0x180000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36380 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36310) get_ref init 0x1000, 1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~1000: 0x0~1000 Blob(0x5557e9c36310 blob([0x180000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9c36380 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36398 in 0x5557e6269880) _discard 0x0~1000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x19000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] generate_transactions: 1:3a5a3f5b:::smithi13140796-23:head, current size is 376832 buffers are {} 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.689+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.689+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 73 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=41, reqid=client.4540.0:61, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d83000 con 0x5557e79cac00 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.689+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85b80 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=41, reqid=client.4540.0:61, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4000 cost 160022 (14 ios * 4000 + 104022 bytes) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4000 onodes 0x5557e9b94f00 shared_blobs 2019-12-07T19:33:58.689+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=41, reqid=client.4540.0:61, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95008) update 1#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): new_size start 376832 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95008) update inline shard 247 bytes from 13 extents 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:ee170468:::smithi13140796-25:head# is 666 (413 bytes onode + 2 bytes spanning blobs + 251 bytes inline extents) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4000 allocated 0x[168000~19000] released 0x[] 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 freelist allocate 0x168000~19000 2019-12-07T19:33:58.689+0000 7f2b72cd7700 20 freelist _xor first_key 0x100000 last_key 0x180000 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): 1:3a5a3f5b:::smithi13140796-23:head resetting hinfo to logical size 376832 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 prepare 2019-12-07T19:33:58.689+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9cc4000 2019-12-07T19:33:58.689+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 74 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=42, reqid=client.4540.0:62, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9d81000 con 0x5557e79cac00 2019-12-07T19:33:58.689+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85ce0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=42, reqid=client.4540.0:62, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.689+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=42, reqid=client.4540.0:62, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.689+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written: {1:3a5a3f5b:::smithi13140796-23:head={}} 2019-12-07T19:33:58.690+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=21'8 tt=0'0 tid=12 reqid=client.4540.0:68 client_op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:33:58.690+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written_set: {1:3a5a3f5b:::smithi13140796-23:head=[]} 2019-12-07T19:33:58.690+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: 1:3a5a3f5b:::smithi13140796-23:head,{} 2019-12-07T19:33:58.690+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:68, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e9d83800 con 0x5557e79cac00 2019-12-07T19:33:58.690+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:68, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e9c7b000 con 0x5557e7519800 2019-12-07T19:33:58.690+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:68, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e763e800 con 0x5557e79ca800 2019-12-07T19:33:58.690+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:68, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e763c000 con 0x5557e79cbc00 2019-12-07T19:33:58.690+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=12, reqid=client.4540.0:68, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x5557e763f800 con 0x5557e79ca000 2019-12-07T19:33:58.690+0000 7f2b744da700 20 earliest_dup_version = 0 2019-12-07T19:33:58.690+0000 7f2b744da700 20 trim 21'8 (21'7) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:68 2019-12-07T19:33:58.673986+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.690+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:68 2019-12-07T19:33:58.673986+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.690+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 lua=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] add_log_entry 21'8 (21'7) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:68 2019-12-07T19:33:58.673986+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.690+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 34 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9d88e00 con 0x5557e79cbc00 2019-12-07T19:33:58.690+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85e40 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.690+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 lua=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log approx pg log length = 8 2019-12-07T19:33:58.690+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.690+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 lua=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.690+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.690+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.690+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9d5af00 seq 15 2019-12-07T19:33:58.690+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=41, reqid=client.4540.0:61, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.690+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85a20 finish 2019-12-07T19:33:58.690+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=41, reqid=client.4540.0:61, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d85b80 prio 127 cost 0 latency 0.000313 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=41, reqid=client.4540.0:61, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=41, reqid=client.4540.0:61, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:61 2019-12-07T19:33:58.668524+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9b10900 seq 12 2019-12-07T19:33:58.690+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 aio_wait 2019-12-07T19:33:58.690+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4000 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.690+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 io_done 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10900 onodes shared_blobs 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.690+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 msgr2=0x5557e7529700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 96 2019-12-07T19:33:58.690+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 msgr2=0x5557e7529700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 prepare 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10900 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 io_done 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d85b80 finish 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.690+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 0x5557e7529700 crc :-1 s=READY pgs=8 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=42, reqid=client.4540.0:62, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=42, reqid=client.4540.0:62, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 prio 127 cost 0 latency 0.000416 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=42, reqid=client.4540.0:62, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=42, reqid=client.4540.0:62, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.690+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 0x5557e7529700 crc :-1 s=READY pgs=8 cs=0 l=0 rx=0 tx=0)._fault initiating reconnect 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:15219530:::smithi13140796-21:head by client.4540.0:62 2019-12-07T19:33:58.668552+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:15219530:::smithi13140796-21:head by client.4540.0:62 2019-12-07T19:33:58.668552+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9b10c00 seq 13 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 3 keys 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 3 keys = 0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:15219530:::smithi13140796-21:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:15219530:::smithi13140796-21:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10c00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10c00 onodes 0x5557e76cb180 shared_blobs 2019-12-07T19:33:58.690+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 unknown :-1 s=BANNER_CONNECTING pgs=17 cs=3 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e76cb288) update 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:15219530:::smithi13140796-21:head# is 700 (485 bytes onode + 2 bytes spanning blobs + 213 bytes inline extents) 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 prepare 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10c00 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 io_done 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 finish 2019-12-07T19:33:58.690+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.690+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.690+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.690+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.690+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 0x5557e7529700 unknown :-1 s=BANNER_CONNECTING pgs=8 cs=1 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.690+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 3 keys 2019-12-07T19:33:58.690+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 3 keys = 0 2019-12-07T19:33:58.690+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.690+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.690+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1407 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.690+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b94f00 had 1 2019-12-07T19:33:58.690+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.690+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.690+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5af00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5af00 onodes 0x5557e9b94280 shared_blobs 2019-12-07T19:33:58.690+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2521 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore.extentmap(0x5557e9b94388) update 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:3a5a3f5b:::smithi13140796-23:head# is 719 (485 bytes onode + 2 bytes spanning blobs + 232 bytes inline extents) 2019-12-07T19:33:58.690+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 2 2019-12-07T19:33:58.690+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4261 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5af00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.690+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 prepare 2019-12-07T19:33:58.690+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb180 had 1 2019-12-07T19:33:58.690+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5af00 2019-12-07T19:33:58.690+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.690+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 io_done 2019-12-07T19:33:58.690+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4284 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4284 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.690+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3ab7f~10bc to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.690+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 lua=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op_applied version 21'8 2019-12-07T19:33:58.690+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3ab7f 0x3ab7f~10bc to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xab7f 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xb7f 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xc3b and padding block with 0x3c5 2019-12-07T19:33:58.691+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e9e2a800 0x5557e75b2100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] eval_repop repgather(0x5557e9cc0000 21'8 rep_tid=12 committed?=0 r=0) 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84000 finish 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3bc3b 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 3 cleaned 0 in 0.000363125s (0.000232962s flush + 0.000130163s kv commit) 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1759 < min_flush_size 524288 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d854a0 prio 63 cost 0 latency 0.015856 osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4000,0x5557e9b10900,0x5557e9b10c00> 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 kv_submitted 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4000 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 finishing 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4000 onodes 0x5557e9b94f00 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7b978 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9b7b978 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7bac8 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9b7bac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7bba8 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7c20 space 0x5557e9b7bba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op: op osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7bc88 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7cb0 space 0x5557e9b7bc88 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7bd68 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7d40 space 0x5557e9b7bd68 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7be48 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd7dd0 space 0x5557e9b7be48 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7bf28 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9b7bf28 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2b908 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e9c2b908 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36018 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aa360 space 0x5557e9c36018 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c360f8 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aa120 space 0x5557e9c360f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c361d8 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aac60 space 0x5557e9c361d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c362b8 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aaea0 space 0x5557e9c362b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36398 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9c36398 0x0~1000 writing nocache) 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4000 done 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: found obc in cache: 0x5557e760c480 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4000 [] 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 kv_submitted 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10900 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: 0x5557e760c480 1:3a5a3f5b:::smithi13140796-23:head rwstate(write n=1 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(21'8 client.4540.0:68 dirty|data_digest s 376832 uv 8 dd 11abb9ec alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e628b200 snapset: 0=[]:{} 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 finishing 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10900 onodes 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10900 done 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(21'8 client.4540.0:68 dirty|data_digest s 376832 uv 8 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10900 [] 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.691+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10c00 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 finishing 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10c00 onodes 0x5557e76cb180 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 done 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.691+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(21'8 client.4540.0:68 dirty|data_digest s 376832 uv 8 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10c00 [] 2019-12-07T19:33:58.691+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 16 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e76b1a00 con 0x5557e79ca000 2019-12-07T19:33:58.691+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=41, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e9d88400 con 0x5557e79cac00 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.691+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4840 prio 196 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d854a0 finish 2019-12-07T19:33:58.691+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.691+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=40, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e9b36000 con 0x5557e79cac00 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=42, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cac00 2019-12-07T19:33:58.691+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.691+0000 7f2b834f8700 10 osd.6 21 ms_handle_connect con 0x5557e7519800 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.691+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b94280 had 1 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85e40 prio 196 cost 0 latency 0.001028 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.691+0000 7f2b95fb3700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x5557e7519800 0x5557e7527600 crc :-1 s=READY pgs=17 cs=3 l=0 rx=0 tx=0).ready entity=osd.2 client_cookie=18704efb08e1a046 server_cookie=2a616824987376d2 in_seq=23 out_seq=23 2019-12-07T19:33:58.691+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e9e2a800 0x5557e75b2100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1782 < min_flush_size 524288 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1782 < min_flush_size 524288 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.691+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85e40 finish 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3bc3b~6f6 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3bc3b 0x3bc3b~6f6 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xbc3b 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xc3b 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x331 and padding block with 0xccf 2019-12-07T19:33:58.691+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3c331 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.691+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000202123s (0.000000085s flush + 0.000202038s kv commit) 2019-12-07T19:33:58.691+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4840 prio 196 cost 0 latency 0.000119 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5af00> 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 kv_submitted 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5af00 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 finishing 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5af00 onodes 0x5557e9b94280 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5af00 done 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5af00 [] 2019-12-07T19:33:58.691+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4840 finish 2019-12-07T19:33:58.691+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f2b957b2700 10 osd.6 21 ms_handle_authentication new session 0x5557e75581a0 con 0x5557e9e2a800 entity osd.4 addr 2019-12-07T19:33:58.691+0000 7f2b957b2700 10 osd.6 21 ms_handle_authentication session 0x5557e75581a0 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.691+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e9e2a800 msgr2=0x5557e75b2100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0)._try_send send error: (32) Broken pipe 2019-12-07T19:33:58.691+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e9e2a800 0x5557e75b2100 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=0 rx=0 tx=0).write auth signature write failed r=-32 ((32) Broken pipe) 2019-12-07T19:33:58.691+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e9e2a800 0x5557e75b2100 crc :-1 s=AUTH_ACCEPTING_SIGN pgs=0 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.691+0000 7f2b834f8700 2 osd.6 21 ms_handle_reset con 0x5557e9e2a800 session 0x5557e75581a0 2019-12-07T19:33:58.691+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 17 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9c77400 con 0x5557e79ca000 2019-12-07T19:33:58.691+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c49a0 prio 196 cost 0 latency 0.000009 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.691+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.691+0000 7f2b834f8700 10 osd.6 21 ms_handle_connect con 0x5557e79cbc00 2019-12-07T19:33:58.691+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x5557e79cbc00 0x5557e7529700 crc :-1 s=READY pgs=8 cs=1 l=0 rx=0 tx=0).ready entity=osd.4 client_cookie=acae43cf971586cc server_cookie=ee5cf5657875a67a in_seq=34 out_seq=34 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c49a0 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.691+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 35 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c76c00 con 0x5557e79cbc00 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.691+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c42c0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c49a0 finish 2019-12-07T19:33:58.691+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.691+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c42c0 prio 196 cost 0 latency 0.000032 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.691+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c42c0 finish 2019-12-07T19:33:58.691+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.693+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 75 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e76b0e00 con 0x5557e79cac00 2019-12-07T19:33:58.693+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 196 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.693+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 76 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36200 con 0x5557e79cac00 2019-12-07T19:33:58.693+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7694b00 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.693+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.693+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.693+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 196 cost 0 latency 0.000061 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.693+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.693+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.693+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.693+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.693+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.693+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 24 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:70, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c7b000 con 0x5557e7519800 2019-12-07T19:33:58.693+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c54a0 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:70, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.693+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:70, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7694b00 prio 196 cost 0 latency 0.000137 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.693+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.693+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7694b00 finish 2019-12-07T19:33:58.693+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.693+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 36 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 112564+0+0 (crc 0 0 0) 0x5557e763c000 con 0x5557e79cbc00 2019-12-07T19:33:58.693+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84b00 prio 127 cost 0 latency 0.000031 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.693+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e9d84b00 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:79, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:79 2019-12-07T19:33:58.687333+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:79 2019-12-07T19:33:58.687333+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.693+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7693500 seq 9 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.3s5_head 0x5557e75b4000) get_onode oid 5#1:cd22b62d:::smithi13140796-27:head# key 0x858000000000000001cd22b6'-!smithi13140796-27!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.693+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1109 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# = 0 2019-12-07T19:33:58.693+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 2 keys 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 2 keys = 0 2019-12-07T19:33:58.693+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.693+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 0x0~1b000 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:cd22b62d:::smithi13140796-27:head# 0x0~1b000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.693+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1b000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.693+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 25 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:71, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9d81800 con 0x5557e7519800 2019-12-07T19:33:58.693+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:71, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.693+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:71, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7693500 14 blobs 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.693+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1b000/1000,1b000,0 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x181000~1b000/1000,1b000,0 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x181000~1b000] 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7a29030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7669e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7a29030 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669e30 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e7a29030) get_ref 0x0~2000 Blob(0x5557e7a29030 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669e30 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e7a29030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7a29030 blob([0x181000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7669e30 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e7669e48 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92ee0 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92e70 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92ee0) get_ref 0x0~2000 Blob(0x5557e9d92ee0 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92e70 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9d92ee0 blob([0x183000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92e70 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d92e88 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92fc0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37570 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92fc0) get_ref 0x0~2000 Blob(0x5557e9d92fc0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37570 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d92fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9d92fc0 blob([0x185000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37570 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37588 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c375e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c375e0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37650 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c375e0) get_ref 0x0~2000 Blob(0x5557e9c375e0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37650 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c375e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9c375e0 blob([0x187000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37650 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37668 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c376c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c376c0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37730 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c376c0) get_ref 0x0~2000 Blob(0x5557e9c376c0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37730 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c376c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9c376c0 blob([0x189000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37730 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37748 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c377a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c377a0 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37810 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c377a0) get_ref 0x0~2000 Blob(0x5557e9c377a0 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37810 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c377a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9c377a0 blob([0x18b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37810 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37828 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c378f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37880 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c378f0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37880) get_ref 0x0~2000 Blob(0x5557e9c37880 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c378f0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9c37880 blob([0x18d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c378f0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37908 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37b90 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37960 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d37b90) get_ref 0x0~2000 Blob(0x5557e9d37b90 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37960 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d37b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9d37b90 blob([0x18f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37960 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37978 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c379d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c379d0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37a40 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c379d0) get_ref 0x0~2000 Blob(0x5557e9c379d0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37a40 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c379d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9c379d0 blob([0x191000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37a40 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37a58 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37ab0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37b20 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37ab0) get_ref 0x0~2000 Blob(0x5557e9c37ab0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37b20 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9c37ab0 blob([0x193000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37b20 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37b38 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37b90 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37c00 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37b90) get_ref 0x0~2000 Blob(0x5557e9c37b90 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37c00 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9c37b90 blob([0x195000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37c00 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37c18 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37c70 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37ce0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37c70) get_ref 0x0~2000 Blob(0x5557e9c37c70 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37ce0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9c37c70 blob([0x197000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37ce0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37cf8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37d50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37dc0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37d50) get_ref 0x0~2000 Blob(0x5557e9c37d50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37dc0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9c37d50 blob([0x199000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37dc0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37dd8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.694+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 178168 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37e30 blob([0x19b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37ea0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37e30) get_ref 0x0~1000 Blob(0x5557e9c37e30 blob([0x19b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37ea0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c37e30) get_ref init 0x1000, 1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~1000: 0x0~1000 Blob(0x5557e9c37e30 blob([0x19b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9c37ea0 sbid 0x0)) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c37eb8 in 0x5557e62696c0) _discard 0x0~1000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1b000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cad88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 0x0~1b000 = 0 2019-12-07T19:33:58.694+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.694+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693500 cost 172214 (15 ios * 4000 + 112214 bytes) 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693500 onodes 0x5557e76cac80 shared_blobs 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cad88) update 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cad88) update inline shard 266 bytes from 14 extents 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:70, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:cd22b62d:::smithi13140796-27:head# is 685 (413 bytes onode + 2 bytes spanning blobs + 270 bytes inline extents) 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:70, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693500 allocated 0x[181000~1b000] released 0x[] 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 freelist allocate 0x181000~1b000 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c54a0 prio 127 cost 0 latency 0.001089 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:70, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 prepare 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7693500 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:70, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:70 2019-12-07T19:33:58.678831+0000 0 ObjectCleanRegions clean_offsets: [393216~18446744073709158399], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7121500 seq 10 2019-12-07T19:33:58.694+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121500 onodes shared_blobs 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 prepare 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121500 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 io_done 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c54a0 finish 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:71, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:71, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000513 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:71, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:71, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:71 2019-12-07T19:33:58.678863+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:71 2019-12-07T19:33:58.678863+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9cc4900 seq 11 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 3 keys 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3c331~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 3 keys = 0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3c331 0x3c331~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.694+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 46 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763e800 con 0x5557e79ca800 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.694+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.694+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e9d84b00 finish 2019-12-07T19:33:58.694+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.694+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4900 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4900 onodes 0x5557e9c7cc80 shared_blobs 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore.extentmap(0x5557e9c7cd88) update 1#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:a8debf31:::smithi13140796-24:head# is 722 (485 bytes onode + 2 bytes spanning blobs + 235 bytes inline extents) 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 prepare 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4900 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 io_done 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.694+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4420 finish 2019-12-07T19:33:58.694+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:76 2019-12-07T19:33:58.684279+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.694+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e9cc4600 seq 10 2019-12-07T19:33:58.694+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4600 onodes shared_blobs 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 prepare 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4600 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 io_done 2019-12-07T19:33:58.694+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9d85340 finish 2019-12-07T19:33:58.694+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xc331 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x331 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x7a2 and padding block with 0x85e 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3c7a2 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000351416s (0.000000129s flush + 0.000351287s kv commit) 2019-12-07T19:33:58.694+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.694+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121500> 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.694+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.694+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:58.694+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121500 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1762 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 finishing 2019-12-07T19:33:58.694+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121500 onodes 2019-12-07T19:33:58.694+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 done 2019-12-07T19:33:58.694+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4900 kv_queued 2019-12-07T19:33:58.694+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121500 [] 2019-12-07T19:33:58.694+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.694+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7cc80 had 1 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2876 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2899 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2899 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3c7a2~b53 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.694+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3c7a2 0x3c7a2~b53 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xc7a2 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x7a2 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.694+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x2f5 and padding block with 0xd0b 2019-12-07T19:33:58.695+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=6, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e9b37400 con 0x5557e7519800 2019-12-07T19:33:58.695+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 47 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9d82000 con 0x5557e79ca800 2019-12-07T19:33:58.695+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d851e0 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.695+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d851e0 prio 127 cost 0 latency 0.000071 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:77 2019-12-07T19:33:58.684306+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:77 2019-12-07T19:33:58.684306+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.695+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7a93800 seq 11 2019-12-07T19:33:58.695+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 3 keys 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# 3 keys = 0 2019-12-07T19:33:58.695+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.695+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93800 onodes 0x5557e9b94c80 shared_blobs 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b94d88) update 5#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:f9bf9ba5:::smithi13140796-26:head# is 741 (485 bytes onode + 2 bytes spanning blobs + 254 bytes inline extents) 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 prepare 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93800 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 io_done 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.695+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d851e0 finish 2019-12-07T19:33:58.695+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 48 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36000 con 0x5557e79ca800 2019-12-07T19:33:58.695+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85e40 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.695+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.695+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 49 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e76b0000 con 0x5557e79ca800 2019-12-07T19:33:58.695+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85600 prio 196 cost 0 latency 0.000006 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.695+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.695+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85e40 prio 196 cost 0 latency 0.000098 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.695+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.695+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85e40 finish 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.695+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85600 prio 196 cost 0 latency 0.000110 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.695+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.695+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85600 finish 2019-12-07T19:33:58.695+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 77 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=43, reqid=client.4540.0:62, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d83800 con 0x5557e79cac00 2019-12-07T19:33:58.695+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=43, reqid=client.4540.0:62, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.695+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=43, reqid=client.4540.0:62, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.695+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=43, reqid=client.4540.0:62, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.695+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=43, reqid=client.4540.0:62, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.695+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000067 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=43, reqid=client.4540.0:62, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.695+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=43, reqid=client.4540.0:62, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.695+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.695+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:15219530:::smithi13140796-21:head by client.4540.0:62 2019-12-07T19:33:58.668552+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.695+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.695+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.695+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.696+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.696+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.696+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7121800 seq 14 2019-12-07T19:33:58.696+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.696+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.696+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.696+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.696+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.696+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.696+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121800 onodes shared_blobs 2019-12-07T19:33:58.696+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.696+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 prepare 2019-12-07T19:33:58.696+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121800 2019-12-07T19:33:58.696+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 io_done 2019-12-07T19:33:58.696+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85760 finish 2019-12-07T19:33:58.696+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3d2f5 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.001406963s (0.000000079s flush + 0.001406884s kv commit) 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4900,0x5557e9cc4600> 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.696+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 kv_submitted 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4900 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1781 < min_flush_size 524288 2019-12-07T19:33:58.696+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 finishing 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4900 onodes 0x5557e9c7cc80 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4900 done 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.696+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4900 [] 2019-12-07T19:33:58.696+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.696+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 kv_submitted 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4600 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b94c80 had 1 2019-12-07T19:33:58.696+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 finishing 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4600 onodes 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4600 done 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93800 kv_submitted 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2895 < min_flush_size 524288 2019-12-07T19:33:58.696+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4600 [] 2019-12-07T19:33:58.696+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.696+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2918 < min_flush_size 524288 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2918 < min_flush_size 524288 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3d2f5~b66 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.696+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3d2f5 0x3d2f5~b66 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xd2f5 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x2f5 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.696+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe5b and padding block with 0x1a5 2019-12-07T19:33:58.696+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=21, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e76b1e00 con 0x5557e79ca800 2019-12-07T19:33:58.696+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=7, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e9ef2200 con 0x5557e7519800 2019-12-07T19:33:58.696+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.696+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.696+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.696+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.696+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 37 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 116660+0+0 (crc 0 0 0) 0x5557e763f800 con 0x5557e79cbc00 2019-12-07T19:33:58.696+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85a20 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.696+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85a20 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7692c00 seq 9 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.bs1_head 0x5557e75b41e0) get_onode oid 1#1:d07269f7:::smithi13140796-28:head# key 0x818000000000000001d07269f721736d'ithi13140796-28!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.696+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1110 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# = 0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1c000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:d07269f7:::smithi13140796-28:head# 0x0~1c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1c000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692c00 14 blobs 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1c000/1000,1c000,0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x19c000~1c000/1000,1c000,0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x19c000~1c000] 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b70690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ba40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b70690 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ba40 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b70690) get_ref 0x0~2000 Blob(0x5557e9b70690 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2ba40 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b70690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9b70690 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2ba40 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9c2ba58 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d93180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d93180 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93260 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d93180) get_ref 0x0~2000 Blob(0x5557e9d93180 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93260 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d93180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9d93180 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93260 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93278 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74d9730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74d9730 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2b60 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74d9730) get_ref 0x0~2000 Blob(0x5557e74d9730 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2b60 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74d9730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e74d9730 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2b60 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9be2b78 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d932d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d932d0 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93340 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d932d0) get_ref 0x0~2000 Blob(0x5557e9d932d0 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93340 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d932d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9d932d0 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93340 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93358 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d933b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d933b0 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93420 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d933b0) get_ref 0x0~2000 Blob(0x5557e9d933b0 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93420 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d933b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9d933b0 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93420 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93438 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b571f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb05b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b571f0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb05b0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b571f0) get_ref 0x0~2000 Blob(0x5557e9b571f0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb05b0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b571f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9b571f0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb05b0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb05c8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb0620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb0620 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0690 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0620) get_ref 0x0~2000 Blob(0x5557e9cb0620 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0690 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9cb0620 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0690 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb06a8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb0700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb0700 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0770 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0700) get_ref 0x0~2000 Blob(0x5557e9cb0700 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0770 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9cb0700 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0770 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0788 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb07e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb07e0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0850 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb07e0) get_ref 0x0~2000 Blob(0x5557e9cb07e0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0850 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb07e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9cb07e0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0850 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0868 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb08c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb08c0 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0930 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb08c0) get_ref 0x0~2000 Blob(0x5557e9cb08c0 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0930 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb08c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9cb08c0 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0930 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0948 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb09a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb09a0 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0a10 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb09a0) get_ref 0x0~2000 Blob(0x5557e9cb09a0 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0a10 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb09a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9cb09a0 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0a10 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0a28 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb0a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb0a80 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0af0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0a80) get_ref 0x0~2000 Blob(0x5557e9cb0a80 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0af0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9cb0a80 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0af0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0b08 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb0b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb0b60 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0bd0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0b60) get_ref 0x0~2000 Blob(0x5557e9cb0b60 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0bd0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9cb0b60 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0bd0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0be8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb0c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb0c40 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0c40) get_ref 0x0~2000 Blob(0x5557e9cb0c40 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9cb0c40 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0cc8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1c000 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.696+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1c000 = 0 2019-12-07T19:33:58.696+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.696+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.697+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.697+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.697+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692c00 cost 176310 (15 ios * 4000 + 116310 bytes) 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692c00 onodes 0x5557e9b95400 shared_blobs 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95508) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95508) update inline shard 269 bytes from 14 extents 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 688 (413 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692c00 allocated 0x[19c000~1c000] released 0x[] 2019-12-07T19:33:58.697+0000 7f2b74cdb700 10 freelist allocate 0x19c000~1c000 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.697+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 prepare 2019-12-07T19:33:58.697+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692c00 2019-12-07T19:33:58.697+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.697+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85a20 finish 2019-12-07T19:33:58.697+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.697+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 aio_wait 2019-12-07T19:33:58.697+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693500 2019-12-07T19:33:58.697+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 io_done 2019-12-07T19:33:58.698+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 38 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 120756+0+0 (crc 0 0 0) 0x5557e9d82800 con 0x5557e79cbc00 2019-12-07T19:33:58.698+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85080 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.698+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.698+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.698+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.698+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.698+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.698+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e9d85080 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.3s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.698+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:85, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.698+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 lcod 21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [21'4 (0'0) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:85 2019-12-07T19:33:58.693730+0000 0 ObjectCleanRegions clean_offsets: [475136~18446744073709076479], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.698+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'4 (0'0) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:85 2019-12-07T19:33:58.693730+0000 0 ObjectCleanRegions clean_offsets: [475136~18446744073709076479], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.698+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.698+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.698+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7692900 seq 10 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.3s5_head 0x5557e75b4000) get_onode oid 5#1:c8ce3553:::smithi13140796-29:head# key 0x858000000000000001c8ce35'S!smithi13140796-29!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.699+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1111 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# = 0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 2 keys 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3de5b 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 2 keys = 0 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.699+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1d000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692900 15 blobs 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1d000/1000,1d000,0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1b8000~1d000/1000,1d000,0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x1b8000~1d000] 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1c70 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1c00 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1c70) get_ref 0x0~2000 Blob(0x5557e9cb1c70 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1c00 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9cb1c70 blob([0x1b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1c00 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb1c18 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1b90 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1b20 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1b90) get_ref 0x0~2000 Blob(0x5557e9cb1b90 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1b20 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9cb1b90 blob([0x1ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1b20 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb1b38 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.002883185s (0.000000066s flush + 0.002883119s kv commit) 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1ab0 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1a40 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1ab0) get_ref 0x0~2000 Blob(0x5557e9cb1ab0 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1a40 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9cb1ab0 blob([0x1bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1a40 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb1a58 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb19d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1397 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb19d0 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1960 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb19d0) get_ref 0x0~2000 Blob(0x5557e9cb19d0 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1960 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb19d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9cb19d0 blob([0x1be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1960 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb1978 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb18f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb18f0 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1880 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb18f0) get_ref 0x0~2000 Blob(0x5557e9cb18f0 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1880 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb18f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cac80 had 1 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9cb18f0 blob([0x1c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1880 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb1898 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1420 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1420 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb17a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1810 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb17a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1810) get_ref 0x0~2000 Blob(0x5557e9cb1810 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb17a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3de5b~58c to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9cb1810 blob([0x1c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb17a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb17b8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3de5b 0x3de5b~58c to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xde5b 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb16c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe5b 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb16c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0540 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x3e7 and padding block with 0xc19 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb16c0) get_ref 0x0~2000 Blob(0x5557e9cb16c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0540 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb16c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9cb16c0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0540 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb0558 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2bb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2bb20 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a930 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c2bb20) get_ref 0x0~2000 Blob(0x5557e9c2bb20 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2a930 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9c2bb20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9c2bb20 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2a930 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c2a948 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1ce0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1d50 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a93800,0x5557e7121800> 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1ce0) get_ref 0x0~2000 Blob(0x5557e9cb1ce0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1d50 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 kv_submitted 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93800 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9cb1ce0 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1d50 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb1d68 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 finishing 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93800 onodes 0x5557e9b94c80 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93800 done 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93800 [] 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1dc0 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1e30 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 kv_submitted 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121800 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1dc0) get_ref 0x0~2000 Blob(0x5557e9cb1dc0 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1e30 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 finishing 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121800 onodes 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9cb1dc0 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1e30 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121800 done 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb1e48 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121800 [] 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1ea0 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1f10 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1ea0) get_ref 0x0~2000 Blob(0x5557e9cb1ea0 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1f10 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9cb1ea0 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1f10 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9cb1f28 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1f80 blob([0x1ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6000 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=43, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e76b1000 con 0x5557e79cac00 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1f80) get_ref 0x0~2000 Blob(0x5557e9cb1f80 blob([0x1ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6000 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9cb1f80 blob([0x1ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb6000 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb6018 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3e3e7 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb60e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6070 blob([0x1d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb60e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.699+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6070) get_ref 0x0~2000 Blob(0x5557e9eb6070 blob([0x1d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb60e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9eb6070 blob([0x1d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb60e0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb60f8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000170228s (0.000004768s flush + 0.000165460s kv commit) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb61c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6150 blob([0x1d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb61c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6150) get_ref 0x0~2000 Blob(0x5557e9eb6150 blob([0x1d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb61c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693500> 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 kv_submitted 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9eb6150 blob([0x1d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb61c0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693500 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb61d8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 finishing 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb62a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693500 onodes 0x5557e76cac80 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6230 blob([0x1d4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb62a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7669e48 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79ab9e0 space 0x5557e7669e48 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6230) get_ref 0x0~1000 Blob(0x5557e9eb6230 blob([0x1d4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb62a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6230) get_ref init 0x1000, 1000 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37588 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7b00 space 0x5557e9c37588 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~1000: 0x0~1000 Blob(0x5557e9eb6230 blob([0x1d4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9eb62a0 sbid 0x0)) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37668 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7a70 space 0x5557e9c37668 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37748 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9c37748 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37828 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9c37828 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37908 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e9c37908 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37978 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9c37978 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37a58 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9c37a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37b38 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9c37b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37c18 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7c20 space 0x5557e9c37c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37cf8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7cb0 space 0x5557e9c37cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37dd8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7d40 space 0x5557e9c37dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37eb8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7dd0 space 0x5557e9c37eb8 0x0~1000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92e88 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd7b90 space 0x5557e9d92e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693500 done 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 prepare 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693500 [] 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.699+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb62b8 in 0x5557e62696c0) _discard 0x0~1000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1d000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9eb4108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 = 0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692900 cost 184406 (16 ios * 4000 + 120406 bytes) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692900 onodes 0x5557e9eb4000 shared_blobs 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9eb4108) update 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9eb4108) update inline shard 285 bytes from 15 extents 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:c8ce3553:::smithi13140796-29:head# is 704 (413 bytes onode + 2 bytes spanning blobs + 289 bytes inline extents) 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692900 allocated 0x[1b8000~1d000] released 0x[] 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 freelist allocate 0x1b8000~1d000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 prepare 2019-12-07T19:33:58.699+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692900 2019-12-07T19:33:58.699+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 aio_wait 2019-12-07T19:33:58.699+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692c00 2019-12-07T19:33:58.699+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 io_done 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1400 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95400 had 1 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1423 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1423 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3e3e7~58f to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3e3e7 0x3e3e7~58f to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xe3e7 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x3e7 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x976 and padding block with 0x68a 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3e976 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.699+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.699+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=22, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e9ef3e00 con 0x5557e79ca800 2019-12-07T19:33:58.699+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000284751s (0.000046690s flush + 0.000238061s kv commit) 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692c00> 2019-12-07T19:33:58.699+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.699+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 kv_submitted 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692c00 2019-12-07T19:33:58.700+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 finishing 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692c00 onodes 0x5557e9b95400 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be2b78 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44000 space 0x5557e9be2b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2ba58 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9cd6360 space 0x5557e9c2ba58 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb05c8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e441b0 space 0x5557e9cb05c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb06a8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44240 space 0x5557e9cb06a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0788 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e442d0 space 0x5557e9cb0788 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0868 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44360 space 0x5557e9cb0868 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0948 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e443f0 space 0x5557e9cb0948 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0a28 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44480 space 0x5557e9cb0a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0b08 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44510 space 0x5557e9cb0b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0be8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e445a0 space 0x5557e9cb0be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0cc8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44630 space 0x5557e9cb0cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93278 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9cd62d0 space 0x5557e9d93278 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93358 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44090 space 0x5557e9d93358 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93438 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44120 space 0x5557e9d93438 0x0~2000 writing nocache) 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692c00 done 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.700+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692c00 [] 2019-12-07T19:33:58.700+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.700+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.699+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.700+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.700+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e9d85080 finish 2019-12-07T19:33:58.700+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.700+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=10, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e765ea00 con 0x5557e79cbc00 2019-12-07T19:33:58.700+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=11, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e70e2a00 con 0x5557e79cbc00 2019-12-07T19:33:58.700+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.700+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 aio_wait 2019-12-07T19:33:58.700+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692900 2019-12-07T19:33:58.700+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 io_done 2019-12-07T19:33:58.700+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.700+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.700+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.701+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 26 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9ef2200 con 0x5557e7519800 2019-12-07T19:33:58.701+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c51e0 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.701+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.701+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 27 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e7683e00 con 0x5557e7519800 2019-12-07T19:33:58.701+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c5600 prio 196 cost 0 latency 0.000004 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c51e0 prio 196 cost 0 latency 0.000044 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=11, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=11 reqid=client.4540.0:67 roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c51e0 finish 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5600 prio 196 cost 0 latency 0.000074 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=12, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=21'8 tt=0'0 tid=12 reqid=client.4540.0:68 client_op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] repop_all_committed: repop tid 12 all committed 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] eval_repop repgather(0x5557e9cc0000 21'8 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] commit: repgather(0x5557e9cc0000 21'8 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.701+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] log_op_stats osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 0 outb 0 lat 0.026048 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] sending reply on osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x5557e9d59680 2019-12-07T19:33:58.701+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(68 smithi13140796-23 [setxattr (56)] v21'8 uv8 ondisk = 0) v8 -- 0x5557e9d59680 con 0x5557e6f87800 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.701+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] publish_stats_to_osd 21:23 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] removing repgather(0x5557e9cc0000 21'8 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] q front is repgather(0x5557e9cc0000 21'8 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.701+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] remove_repop repgather(0x5557e9cc0000 21'8 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] requeue_op 0x5557e9d854a0 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=21'8 tt=0'0 tid=12 reqid=client.4540.0:68 client_op=osd_op(client.4540.0:68 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3a5a3f5b:::smithi13140796-23:head=[]}) 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: starting commit on Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=13 reqid=client.4540.0:68 roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: op: Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=13 reqid=client.4540.0:68 roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.701+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9d80800 con 0x5557e79cac00 2019-12-07T19:33:58.701+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=13, reqid=client.4540.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e7a0e000 con 0x5557e7519800 2019-12-07T19:33:58.701+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=13, reqid=client.4540.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e70a7000 con 0x5557e79ca800 2019-12-07T19:33:58.701+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e79e2800 con 0x5557e79cbc00 2019-12-07T19:33:58.701+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:68, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9d80000 con 0x5557e79ca000 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] append_log log((0'0,21'8], crt=21'7) [] 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] rollforward: entry=21'8 (21'7) modify 1:3a5a3f5b:::smithi13140796-23:head by client.4540.0:68 2019-12-07T19:33:58.673986+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log approx pg log length = 8 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.701+0000 7f2b784e2700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9cc5200 seq 16 2019-12-07T19:33:58.701+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5200 onodes shared_blobs 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 prepare 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5200 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 io_done 2019-12-07T19:33:58.701+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5600 finish 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d854a0 prio 63 cost 0 latency 0.026356 osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op: op osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_object_context: found obc in cache: 0x5557e760c480 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_object_context: 0x5557e760c480 1:3a5a3f5b:::smithi13140796-23:head rwstate(none n=0 w=0) oi: 1:3a5a3f5b:::smithi13140796-23:head(21'8 client.4540.0:68 dirty|data_digest s 376832 uv 8 dd 11abb9ec alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e628b200 snapset: 0=[]:{} 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] find_object_context 1:3a5a3f5b:::smithi13140796-23:head @head oi=1:3a5a3f5b:::smithi13140796-23:head(21'8 client.4540.0:68 dirty|data_digest s 376832 uv 8 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.701+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op oi 1:3a5a3f5b:::smithi13140796-23:head(21'8 client.4540.0:68 dirty|data_digest s 376832 uv 8 dd 11abb9ec alloc_hint [0 0 0]) 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op obc obc(1:3a5a3f5b:::smithi13140796-23:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 0x5557e9b14300 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] ov 21'8 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [read 0~1] 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_read 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] async_read noted for 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_all_avail_shards: checking acting 6(0) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] start_read_op: starting ReadOp(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_read_op: starting read ReadOp(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.701+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e7606500 con 0x5557e79cac00 2019-12-07T19:33:58.701+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9d59180 con 0x5557e79cbc00 2019-12-07T19:33:58.701+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9d58c80 con 0x5557e79ca000 2019-12-07T19:33:58.701+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9d58f00 con 0x5557e62f5400 2019-12-07T19:33:58.701+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x5557e9d58f00 con 0x5557e62f5400 2019-12-07T19:33:58.701+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e9d84b00 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_read_op: started ReadOp(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,6(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),4(2),5(1),6(0)) 2019-12-07T19:33:58.701+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d854a0 finish 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.701+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84b00 prio 127 cost 0 latency 0.000048 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.701+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.701+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=0,4096,0}, subchunks={1:3a5a3f5b:::smithi13140796-23:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.701+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.701+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~1000 2019-12-07T19:33:58.701+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x17000 (94208) 2019-12-07T19:33:58.701+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.701+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b704d0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b703f0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.701+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b704d0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b703f0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.701+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.701+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.701+0000 7f2b957b2700 0 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 msgr2=0x5557e7528100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 msgr2=0x5557e7528100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 92 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 msgr2=0x5557e7528100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=READY pgs=8 cs=1 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=READY pgs=8 cs=1 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.702+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b704d0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b703f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.702+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9b70408 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.702+0000 7f2b744da700 20 _add level 1 near 0 on buffer(0x5557e9cd6900 space 0x5557e9b70408 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.702+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 0x0~1000 = 4096 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_hash_info: Getting attr on 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_hash_info: not in cache 1:3a5a3f5b:::smithi13140796-23:head 2019-12-07T19:33:58.702+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_hash_info: found on disk, size 94208 2019-12-07T19:33:58.702+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key 2019-12-07T19:33:58.702+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3a5a3f5b:::smithi13140796-23:head# hinfo_key = 0 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 39 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 ==== 4261+0+0 (crc 0 0 0) 0x5557e9d58c80 con 0x5557e79cbc00 2019-12-07T19:33:58.702+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 -- 0x5557e9d59b80 con 0x5557e62f5400 2019-12-07T19:33:58.702+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84b00 finish 2019-12-07T19:33:58.702+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.702+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x5557e9d59b80 con 0x5557e62f5400 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.702+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e9d85b80 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.702+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000034 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=14, attrs_read=0) 2019-12-07T19:33:58.702+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,6(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),5(1),6(0)) 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.702+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.702+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85b80 prio 127 cost 0 latency 0.000081 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.702+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.702+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=14, attrs_read=0) 2019-12-07T19:33:58.702+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096, 6(0),4096]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,6(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3),5(1)) 2019-12-07T19:33:58.702+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85b80 finish 2019-12-07T19:33:58.702+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.702+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e9e2bc00 0x5557e75b2680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 50 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 128948+0+0 (crc 0 0 0) 0x5557e70a7000 con 0x5557e79ca800 2019-12-07T19:33:58.702+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695600 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.702+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.702+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1416 < min_flush_size 524288 2019-12-07T19:33:58.702+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4000 had 1 2019-12-07T19:33:58.702+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> conn(0x5557e9e2bc00 0x5557e75b2680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.702+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1439 < min_flush_size 524288 2019-12-07T19:33:58.702+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1439 < min_flush_size 524288 2019-12-07T19:33:58.702+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.702+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3e976~59f to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.702+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3e976 0x3e976~59f to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.702+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.702+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xe976 2019-12-07T19:33:58.702+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x976 2019-12-07T19:33:58.702+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.702+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xf15 and padding block with 0xeb 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7695600 prio 127 cost 0 latency 0.000167 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.702+0000 7f2b957b2700 10 osd.6 21 ms_handle_authentication new session 0x5557e62956c0 con 0x5557e9e2bc00 entity osd.5 addr 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:91, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'6], crt=21'6) [21'7 (0'0) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:91 2019-12-07T19:33:58.699920+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.702+0000 7f2b957b2700 10 osd.6 21 ms_handle_authentication session 0x5557e62956c0 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] add_log_entry 21'7 (0'0) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:91 2019-12-07T19:33:58.699920+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.702+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7693200 seq 12 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) get_onode oid 1#1:6ad15c4c:::smithi13140796-31:head# key 0x8180000000000000016ad15c'L!smithi13140796-31!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.702+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1112 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# = 0 2019-12-07T19:33:58.702+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 2 keys 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 2 keys = 0 2019-12-07T19:33:58.702+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.702+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 0x0~1f000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:6ad15c4c:::smithi13140796-31:head# 0x0~1f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1f000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7693200 16 blobs 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1f000/1000,1f000,0 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1d5000~1f000/1000,1f000,0 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x1d5000~1f000] 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e9e2bc00 0x5557e75b2680 crc :-1 s=SESSION_ACCEPTING pgs=10 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x5557e79ca000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be3b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be3b20 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56bd0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e9e2bc00 0x5557e75b2680 crc :-1 s=SESSION_ACCEPTING pgs=10 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be3b20) get_ref 0x0~2000 Blob(0x5557e9be3b20 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56bd0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be3b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9be3b20 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b56bd0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9b56be8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b834f8700 2 osd.6 21 ms_handle_reset con 0x5557e9e2bc00 session 0x5557e62956c0 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669dc0 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37f10 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e7669dc0) get_ref 0x0~2000 Blob(0x5557e7669dc0 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37f10 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e7669dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e7669dc0 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37f10 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9d37f28 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d375e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37730 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d375e0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37730) get_ref 0x0~2000 Blob(0x5557e9d37730 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d375e0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x5557e79ca000 0x5557e7528100 unknown :-1 s=READY pgs=10 cs=2 l=0 rx=0 tx=0).ready entity=osd.5 client_cookie=fce9122ddfb0a0db server_cookie=6547c7d58203b459 in_seq=17 out_seq=21 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9d37730 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d375e0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9d375f8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37570 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37650 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37570) get_ref 0x0~2000 Blob(0x5557e9d37570 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37650 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9d37570 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37650 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9d37668 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d376c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d377a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d376c0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d377a0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d376c0) get_ref 0x0~2000 Blob(0x5557e9d376c0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d377a0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d376c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9d376c0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d377a0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9d377b8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d378f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37810 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d378f0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37810) get_ref 0x0~2000 Blob(0x5557e9d37810 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d378f0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9d37810 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d378f0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9d37908 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d379d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d379d0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37a40 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d379d0) get_ref 0x0~2000 Blob(0x5557e9d379d0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37a40 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d379d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9d379d0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37a40 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9d37a58 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37960 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37880) get_ref 0x0~2000 Blob(0x5557e9d37880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37960 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9d37880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37960 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9d37978 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 78 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d80000 con 0x5557e79cac00 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be28c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be39d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be28c0 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be39d0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be28c0) get_ref 0x0~2000 Blob(0x5557e9be28c0 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be39d0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9be28c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e76958c0 prio 127 cost 0 latency 0.000091 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9be28c0 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be39d0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9be39e8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb2620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb2620 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7030 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9cb2620) get_ref 0x0~2000 Blob(0x5557e9cb2620 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7030 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9cb2620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9cb2620 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb7030 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb7048 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb70a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb70a0 blob([0x1e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7110 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb70a0) get_ref 0x0~2000 Blob(0x5557e9eb70a0 blob([0x1e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7110 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb70a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9eb70a0 blob([0x1e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb7110 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb7128 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb7180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb71f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb7180 blob([0x1eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb71f0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7180) get_ref 0x0~2000 Blob(0x5557e9eb7180 blob([0x1eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb71f0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9eb7180 blob([0x1eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb71f0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb7208 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 18 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x5557e7606500 con 0x5557e79ca000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb7260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb72d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695340 prio 127 cost 0 latency 0.000012 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb7260 blob([0x1ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb72d0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7260) get_ref 0x0~2000 Blob(0x5557e9eb7260 blob([0x1ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb72d0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9eb7260 blob([0x1ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb72d0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb72e8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb7340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb73b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb7340 blob([0x1ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb73b0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7340) get_ref 0x0~2000 Blob(0x5557e9eb7340 blob([0x1ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb73b0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9eb7340 blob([0x1ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb73b0 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb73c8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb7420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb7420 blob([0x1f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7490 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695340 prio 127 cost 0 latency 0.000037 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7420) get_ref 0x0~2000 Blob(0x5557e9eb7420 blob([0x1f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7490 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.702+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9eb7420 blob([0x1f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb7490 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb74a8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=14, attrs_read=0) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb7500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb7500 blob([0x1f3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7570 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7500) get_ref 0x0~1000 Blob(0x5557e9eb7500 blob([0x1f3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7570 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7500) get_ref init 0x1000, 1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~1000: 0x0~1000 Blob(0x5557e9eb7500 blob([0x1f3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9eb7570 sbid 0x0)) 2019-12-07T19:33:58.702+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e76958c0 prio 127 cost 0 latency 0.000189 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb7588 in 0x5557e6268fc0) _discard 0x0~1000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1f000 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(2),4096, 5(1),4096, 6(0),4096]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,6(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=1(3)) 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb4b08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.702+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695340 finish 2019-12-07T19:33:58.702+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 0x0~1f000 = 0 2019-12-07T19:33:58.702+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.702+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.702+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.702+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.702+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.702+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.702+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3ef15 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.703+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.703+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693200 cost 196598 (17 ios * 4000 + 128598 bytes) 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693200 onodes 0x5557e9eb4a00 shared_blobs 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb4b08) update 1#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:33:58.703+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb4b08) update inline shard 304 bytes from 16 extents 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:6ad15c4c:::smithi13140796-31:head# is 723 (413 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.002822925s (0.002153730s flush + 0.000669195s kv commit) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693200 allocated 0x[1d5000~1f000] released 0x[] 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692900> 2019-12-07T19:33:58.703+0000 7f2b734d8700 10 freelist allocate 0x1d5000~1f000 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.703+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 kv_submitted 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692900 2019-12-07T19:33:58.703+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e9cc5800 seq 14 2019-12-07T19:33:58.703+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 finishing 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692900 onodes 0x5557e9eb4000 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2a948 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44c60 space 0x5557e9c2a948 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0558 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44bd0 space 0x5557e9cb0558 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 prepare 2019-12-07T19:33:58.703+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.703+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7693200 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb17b8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e9cb17b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1898 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9cb1898 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1978 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79aaea0 space 0x5557e9cb1978 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1a58 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79aac60 space 0x5557e9cb1a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1b38 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79aa120 space 0x5557e9cb1b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1c18 in 0x5557e62696c0) _finish_write discard buffer(0x5557e79aa360 space 0x5557e9cb1c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1d68 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44cf0 space 0x5557e9cb1d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1e48 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44d80 space 0x5557e9cb1e48 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1f28 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44e10 space 0x5557e9cb1f28 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb6018 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44ea0 space 0x5557e9eb6018 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb60f8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44f30 space 0x5557e9eb60f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb61d8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44fc0 space 0x5557e9eb61d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb62b8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e45050 space 0x5557e9eb62b8 0x0~1000 writing nocache) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 done 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.703+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.703+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692900 [] 2019-12-07T19:33:58.703+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3ef15~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.703+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.703+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5800 onodes shared_blobs 2019-12-07T19:33:58.703+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3ef15 0x3ef15~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.703+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xef15 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xf15 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.703+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x386 and padding block with 0xc7a 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 prepare 2019-12-07T19:33:58.703+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5800 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 io_done 2019-12-07T19:33:58.703+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e76958c0 finish 2019-12-07T19:33:58.703+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.703+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=12, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e76b1e00 con 0x5557e79cbc00 2019-12-07T19:33:58.703+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.703+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.703+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7695600 finish 2019-12-07T19:33:58.703+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.703+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 28 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e626fa00 con 0x5557e7519800 2019-12-07T19:33:58.703+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c58c0 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.703+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.703+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 79 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9d80800 con 0x5557e79cac00 2019-12-07T19:33:58.703+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695b80 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.703+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.703+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.703+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.703+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.703+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 29 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e7a0e000 con 0x5557e7519800 2019-12-07T19:33:58.703+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.703+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.703+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.703+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c58c0 prio 196 cost 0 latency 0.000048 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.703+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.703+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 30 ==== MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=10, to_read={1:a8debf31:::smithi13140796-24:head=0,4096,0}, subchunks={1:a8debf31:::smithi13140796-24:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9b7d180 con 0x5557e7519800 2019-12-07T19:33:58.703+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c58c0 finish 2019-12-07T19:33:58.703+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=10, to_read={1:a8debf31:::smithi13140796-24:head=0,4096,0}, subchunks={1:a8debf31:::smithi13140796-24:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.703+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.703+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=10, to_read={1:a8debf31:::smithi13140796-24:head=0,4096,0}, subchunks={1:a8debf31:::smithi13140796-24:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.704+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 51 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e765ea00 con 0x5557e79ca800 2019-12-07T19:33:58.704+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695e40 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000091 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.704+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=9, reqid=client.4540.0:71, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.704+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 52 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e2800 con 0x5557e79ca800 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:71 2019-12-07T19:33:58.678863+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.704+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695760 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.704+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.704+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.704+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e7695b80 prio 127 cost 0 latency 0.000180 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.704+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695e40 prio 196 cost 0 latency 0.000071 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9b10f00 seq 12 2019-12-07T19:33:58.704+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695e40 finish 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.704+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.704+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e7695760 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10f00 onodes shared_blobs 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 prepare 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 io_done 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c5e40 finish 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.704+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=10, to_read={1:a8debf31:::smithi13140796-24:head=0,4096,0}, subchunks={1:a8debf31:::smithi13140796-24:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:77 2019-12-07T19:33:58.684306+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=10, to_read={1:a8debf31:::smithi13140796-24:head=0,4096,0}, subchunks={1:a8debf31:::smithi13140796-24:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7a93b00 seq 15 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.704+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 3 keys 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000209 MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=10, to_read={1:a8debf31:::smithi13140796-24:head=0,4096,0}, subchunks={1:a8debf31:::smithi13140796-24:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 3 keys = 0 2019-12-07T19:33:58.704+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.704+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.704+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=10, to_read={1:a8debf31:::smithi13140796-24:head=0,4096,0}, subchunks={1:a8debf31:::smithi13140796-24:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.704+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.704+0000 7f2b73cd9700 25 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e9d5b500 seq 12 2019-12-07T19:33:58.704+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 0x0~1000 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x18000 (98304) 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.704+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9be27e0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70700 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9be27e0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70700 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93b00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93b00 onodes 0x5557e9b94f00 shared_blobs 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95008) update 1#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5b500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5b500 onodes shared_blobs 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:ee170468:::smithi13140796-25:head# is 738 (485 bytes onode + 2 bytes spanning blobs + 251 bytes inline extents) 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5b500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 prepare 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5b500 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 io_done 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.704+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 prepare 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93b00 2019-12-07T19:33:58.704+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e7695760 finish 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 io_done 2019-12-07T19:33:58.704+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.704+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e7695b80 finish 2019-12-07T19:33:58.704+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.704+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 80 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 ==== 4261+0+0 (crc 0 0 0) 0x5557e9d59180 con 0x5557e79cac00 2019-12-07T19:33:58.704+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5340 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.704+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.704+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5340 prio 127 cost 0 latency 0.000053 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=14, attrs_read=0) 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_read_reply Complete: ReadOp(tid=14, to_read={1:3a5a3f5b:::smithi13140796-23:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:3a5a3f5b:::smithi13140796-23:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 5(1),4096, 6(0),4096]))}, priority=127, obj_to_source={1:3a5a3f5b:::smithi13140796-23:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:3a5a3f5b:::smithi13140796-23:head,4(2)=1:3a5a3f5b:::smithi13140796-23:head,5(1)=1:3a5a3f5b:::smithi13140796-23:head,6(0)=1:3a5a3f5b:::smithi13140796-23:head}, in_progress=) 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] objects_read_async_cb: got: {1:3a5a3f5b:::smithi13140796-23:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 0x5557e9b14300 2019-12-07T19:33:58.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 1:3a5a3f5b:::smithi13140796-23:head [read 0~1 out=1b] ov 21'8 2019-12-07T19:33:58.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op 1:3a5a3f5b:::smithi13140796-23:head [read 0~1 out=1b] 2019-12-07T19:33:58.704+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx alloc reply 0x5557e9d59b80 result 0 2019-12-07T19:33:58.704+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.704+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] log_op_stats osd_op(client.4540.0:69 1.cs0 1:3a5a3f5b:::smithi13140796-23:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 1 lat 0.029261 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.704+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] publish_stats_to_osd 21:24 2019-12-07T19:33:58.704+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(69 smithi13140796-23 [read 0~1 out=1b] v0'0 uv8 ondisk = 0) v8 -- 0x5557e9d59b80 con 0x5557e6f87800 2019-12-07T19:33:58.704+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5340 finish 2019-12-07T19:33:58.704+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.705+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9be27e0 blob([0x136000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70700 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.705+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b70718 in 0x5557e6269180) _discard 0x0~1000 2019-12-07T19:33:58.705+0000 7f2b73cd9700 20 _add level 1 near 0 on buffer(0x5557e9cd73b0 space 0x5557e9b70718 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.705+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 0x0~1000 = 4096 2019-12-07T19:33:58.705+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.705+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:a8debf31:::smithi13140796-24:head 2019-12-07T19:33:58.705+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:a8debf31:::smithi13140796-24:head 2019-12-07T19:33:58.705+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:33:58.705+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 98304 2019-12-07T19:33:58.705+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# hinfo_key 2019-12-07T19:33:58.705+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.5s1_head 1#1:a8debf31:::smithi13140796-24:head# hinfo_key = 0 2019-12-07T19:33:58.705+0000 7f2b73cd9700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.5s0 21/18 ECSubReadReply(tid=10, attrs_read=0)) v2 -- 0x5557e9b94000 con 0x5557e7519800 2019-12-07T19:33:58.705+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e7694b00 finish 2019-12-07T19:33:58.705+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.705+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 81 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9ef3e00 con 0x5557e79cac00 2019-12-07T19:33:58.705+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85e40 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.705+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.705+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.705+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.705+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.705+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.705+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d85e40 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.705+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.705+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d85e40 finish 2019-12-07T19:33:58.705+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x3f386 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 40 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9ba0000 con 0x5557e79cbc00 2019-12-07T19:33:58.707+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.707+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.004555589s (0.000000091s flush + 0.004555498s kv commit) 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 2 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2228 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc5200> 2019-12-07T19:33:58.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 kv_submitted 2019-12-07T19:33:58.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5200 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 finishing 2019-12-07T19:33:58.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5200 onodes 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3194 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5200 done 2019-12-07T19:33:58.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.707+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5200 [] 2019-12-07T19:33:58.707+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3353 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5131 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b94f00 had 1 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.707+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5154 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5154 < min_flush_size 524288 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x3f386~1422 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x3f386 0x3f386~1422 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.659295+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _allocate len 0x7a8 from 0 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _allocate len 0x7a8 from 1 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,acd170000 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd170000~10000/10000,10000,acd170000 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x50000 to include allocated 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 26 (was clean) 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0xf386 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x386 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x7a8 and padding block with 0x858 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x50000 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x407a8 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000308 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (26) on file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]), flushing log 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 26 len 0x49 crc 0x145b8328) 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x19000~1000 to file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.707+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x19000 0x19000~1000 to file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.707+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x19000 2019-12-07T19:33:58.707+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.707+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 41 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e76b1000 con 0x5557e79cbc00 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7630300 seq 10 2019-12-07T19:33:58.707+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84000 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.707+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.707+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.707+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.707+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.707+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.707+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630300 onodes shared_blobs 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 prepare 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630300 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 io_done 2019-12-07T19:33:58.707+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84000 prio 196 cost 0 latency 0.000034 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.707+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85600 finish 2019-12-07T19:33:58.707+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.707+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.707+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84000 finish 2019-12-07T19:33:58.707+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.708+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 19 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e70e2a00 con 0x5557e79ca000 2019-12-07T19:33:58.708+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85b80 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.708+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.708+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 42 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9b9e000 con 0x5557e79cbc00 2019-12-07T19:33:58.708+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d854a0 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.708+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.708+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.708+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.708+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.708+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e9d854a0 prio 127 cost 0 latency 0.000070 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.3s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.708+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d85b80 prio 196 cost 0 latency 0.000106 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.708+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=13, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:80, at_version=21'5, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.708+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_finish_rmw: Op(1:3a5a3f5b:::smithi13140796-23:head v=0'0 tt=0'0 tid=13 reqid=client.4540.0:68 roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.708+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'4], crt=21'2) [21'5 (21'3) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:80 2019-12-07T19:33:58.687362+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.708+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:33:58.708+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d85b80 finish 2019-12-07T19:33:58.708+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'4 active mbc={}] add_log_entry 21'5 (21'3) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:80 2019-12-07T19:33:58.687362+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.3s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'3 lcod 21'4 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:79 2019-12-07T19:33:58.687333+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'3 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'3 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.708+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7665200 seq 11 2019-12-07T19:33:58.708+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 3 keys 2019-12-07T19:33:58.708+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 aio_wait 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 3 keys = 0 2019-12-07T19:33:58.708+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693200 2019-12-07T19:33:58.708+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.708+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.708+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 io_done 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.708+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x1a000 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.708+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.708+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665200 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665200 onodes 0x5557e76cac80 shared_blobs 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cad88) update 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:cd22b62d:::smithi13140796-27:head# is 757 (485 bytes onode + 2 bytes spanning blobs + 270 bytes inline extents) 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 prepare 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665200 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 io_done 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'3 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.708+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e9d854a0 finish 2019-12-07T19:33:58.708+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.708+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000109 2019-12-07T19:33:58.708+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.708+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 43 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9b9f000 con 0x5557e79cbc00 2019-12-07T19:33:58.708+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d858c0 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.708+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d858c0 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.708+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7631b00 seq 11 2019-12-07T19:33:58.708+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 3 keys 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 3 keys = 0 2019-12-07T19:33:58.708+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.708+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631b00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631b00 onodes 0x5557e9b95400 shared_blobs 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95508) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 760 (485 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 prepare 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631b00 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 io_done 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.708+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d858c0 finish 2019-12-07T19:33:58.708+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.709+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 82 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=48, reqid=client.4540.0:94, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 133044+0+0 (crc 0 0 0) 0x5557e7a11000 con 0x5557e79cac00 2019-12-07T19:33:58.709+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84b00 prio 127 cost 0 latency 0.000033 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=48, reqid=client.4540.0:94, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.709+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=48, reqid=client.4540.0:94, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=48, reqid=client.4540.0:94, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=48, reqid=client.4540.0:94, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d84b00 prio 127 cost 0 latency 0.000069 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=48, reqid=client.4540.0:94, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=48, reqid=client.4540.0:94, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'6], crt=21'6) [21'7 (0'0) modify 1:10779340:::smithi13140796-32:head by client.4540.0:94 2019-12-07T19:33:58.703907+0000 0 ObjectCleanRegions clean_offsets: [524288~18446744073709027327], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] add_log_entry 21'7 (0'0) modify 1:10779340:::smithi13140796-32:head by client.4540.0:94 2019-12-07T19:33:58.703907+0000 0 ObjectCleanRegions clean_offsets: [524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9b10300 seq 15 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:10779340:::smithi13140796-32:head# key 0x858000000000000001107793'@!smithi13140796-32!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1113 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:10779340:::smithi13140796-32:head# = 0 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:10779340:::smithi13140796-32:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:10779340:::smithi13140796-32:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 0x0~20000 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:10779340:::smithi13140796-32:head# 0x0~20000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.709+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~20000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10300 16 blobs 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 fbmap_alloc 0x5557e6282e00 allocate 0x20000/1000,20000,0 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1f4000~20000/1000,20000,0 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x1f4000~20000] 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2b9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2b9d0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bd50 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c2b9d0) get_ref 0x0~2000 Blob(0x5557e9c2b9d0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bd50 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c2b9d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9c2b9d0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2bd50 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9c2bd68 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 53 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 137140+0+0 (crc 0 0 0) 0x5557e9c79000 con 0x5557e79ca800 2019-12-07T19:33:58.710+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.710+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37030 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37180 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c37030) get_ref 0x0~2000 Blob(0x5557e9c37030 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37180 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c37030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9c37030 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37180 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9c37198 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37110 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37260 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c37110) get_ref 0x0~2000 Blob(0x5557e9c37110 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37260 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c37110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9c37110 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37260 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9c37278 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c372d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c372d0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37340 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c372d0) get_ref 0x0~2000 Blob(0x5557e9c372d0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37340 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c372d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9c372d0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37340 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9c37358 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c373b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c373b0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37420 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c373b0) get_ref 0x0~2000 Blob(0x5557e9c373b0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37420 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c373b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9c373b0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37420 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9c37438 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c37490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c37490 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37500 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c37490) get_ref 0x0~2000 Blob(0x5557e9c37490 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c37500 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c37490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:97 2019-12-07T19:33:58.707663+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9c37490 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c37500 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9c37518 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd4000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd4000 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4070 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4000) get_ref 0x0~2000 Blob(0x5557e9dd4000 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4070 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9dd4000 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd4070 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd4088 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:97 2019-12-07T19:33:58.707663+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd40e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd40e0 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4150 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd40e0) get_ref 0x0~2000 Blob(0x5557e9dd40e0 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4150 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd40e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9dd40e0 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd4150 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd4168 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd41c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd41c0 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4230 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd41c0) get_ref 0x0~2000 Blob(0x5557e9dd41c0 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4230 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd41c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9dd41c0 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd4230 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd4248 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e9d5b800 seq 13 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd42a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd42a0 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4310 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd42a0) get_ref 0x0~2000 Blob(0x5557e9dd42a0 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4310 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd42a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.fs5_head 0x5557e75e32c0) get_onode oid 5#1:fc3a86a2:::smithi13140796-33:head# key 0x858000000000000001fc3a86a221736d'ithi13140796-33!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9dd42a0 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd4310 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd4328 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1114 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# = 0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2 keys 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd4380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd43f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd4380 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd43f0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 2 keys = 0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4380) get_ref 0x0~2000 Blob(0x5557e9dd4380 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd43f0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9dd4380 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd43f0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd4408 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~21000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd4460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd44d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd4460 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd44d0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4460) get_ref 0x0~2000 Blob(0x5557e9dd4460 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd44d0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9dd4460 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd44d0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd44e8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9d5b800 17 blobs 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd4540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd45b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd4540 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd45b0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4540) get_ref 0x0~2000 Blob(0x5557e9dd4540 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd45b0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9dd4540 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd45b0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd45c8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd4620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd4620 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4690 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4620) get_ref 0x0~2000 Blob(0x5557e9dd4620 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4690 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9dd4620 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd4690 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd46a8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd4700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd4700 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4770 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4700) get_ref 0x0~2000 Blob(0x5557e9dd4700 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4770 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd4700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9dd4700 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd4770 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd4788 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd47e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd47e0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4850 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd47e0) get_ref 0x0~2000 Blob(0x5557e9dd47e0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4850 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd47e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9dd47e0 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd4850 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd4868 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 fbmap_alloc 0x5557e6282e00 allocate 0x21000/1000,21000,0 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x20000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x214000~21000/1000,21000,0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x214000~21000] 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9eb5788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb3810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2be30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb3810 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2be30 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 0x0~20000 = 0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9cb3810) get_ref 0x0~2000 Blob(0x5557e9cb3810 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2be30 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9cb3810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9cb3810 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2be30 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9c2be48 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37b20 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9d37b20) get_ref 0x0~2000 Blob(0x5557e9d37b20 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9d37b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9d37b20 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9d37c18 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b57c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b57c00 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5260 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9b57c00) get_ref 0x0~2000 Blob(0x5557e9b57c00 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5260 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9b57c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 26 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9b57c00 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5260 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5278 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd52d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd52d0 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5340 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd52d0) get_ref 0x0~2000 Blob(0x5557e9dd52d0 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5340 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd52d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9dd52d0 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5340 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5358 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd53b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd53b0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5420 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd53b0) get_ref 0x0~2000 Blob(0x5557e9dd53b0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5420 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd53b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9dd53b0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5420 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5438 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10300 cost 200694 (17 ios * 4000 + 132694 bytes) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5490 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5500 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 4 cleaned 0 in 0.002930739s (0.000000099s flush + 0.002930640s kv commit) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5490) get_ref 0x0~2000 Blob(0x5557e9dd5490 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5500 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10300 onodes 0x5557e9eb5680 shared_blobs 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc5800,0x5557e9b10f00,0x5557e9d5b500,0x5557e7a93b00> 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9dd5490 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5500 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5518 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5800 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9eb5788) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd55e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 finishing 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5570 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd55e0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5800 onodes 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9eb5788) update inline shard 307 bytes from 16 extents 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5800 done 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93b00 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5570) get_ref 0x0~2000 Blob(0x5557e9dd5570 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd55e0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5800 [] 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9dd5570 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd55e0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd55f8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd56c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10f00 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5650 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd56c0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5650) get_ref 0x0~2000 Blob(0x5557e9dd5650 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd56c0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 finishing 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10f00 onodes 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 done 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9dd5650 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd56c0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10f00 [] 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd56d8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5b500 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 finishing 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5b500 onodes 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5b500 done 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 2 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5b800 prepare 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5b500 [] 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2549 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93b00 2019-12-07T19:33:58.710+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=9, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e7a9b400 con 0x5557e7519800 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 finishing 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93b00 onodes 0x5557e9b94f00 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93b00 done 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93b00 [] 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 726 (413 bytes onode + 2 bytes spanning blobs + 311 bytes inline extents) 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4a00 had 1 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10300 allocated 0x[1f4000~20000] released 0x[] 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.710+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 freelist allocate 0x1f4000~20000 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 20 freelist _xor first_key 0x180000 last_key 0x200000 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4346 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cac80 had 1 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 prepare 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6146 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b10300 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95400 had 1 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6169 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6169 < min_flush_size 524288 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x407a8~1819 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x407a8 0x407a8~1819 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x107a8 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x7a8 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xfc1 and padding block with 0x3f 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd57a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5730 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd57a0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5730) get_ref 0x0~2000 Blob(0x5557e9dd5730 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd57a0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9dd5730 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd57a0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd57b8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x41fc1 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5810 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5880 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.710+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5810) get_ref 0x0~2000 Blob(0x5557e9dd5810 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5880 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9dd5810 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5880 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5898 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd58f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd58f0 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5960 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd58f0) get_ref 0x0~2000 Blob(0x5557e9dd58f0 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5960 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd58f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9dd58f0 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5960 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5978 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd59d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd59d0 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5a40 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd59d0) get_ref 0x0~2000 Blob(0x5557e9dd59d0 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5a40 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd59d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9dd59d0 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5a40 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5a58 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5ab0 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5b20 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5ab0) get_ref 0x0~2000 Blob(0x5557e9dd5ab0 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5b20 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9dd5ab0 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5b20 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5b38 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5b90 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5b90) get_ref 0x0~2000 Blob(0x5557e9dd5b90 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9dd5b90 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5c18 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 4 cleaned 0 in 0.000227681s (0.000004709s flush + 0.000222972s kv commit) 2019-12-07T19:33:58.710+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5c70 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5ce0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5c70) get_ref 0x0~2000 Blob(0x5557e9dd5c70 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5ce0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630300,0x5557e7693200,0x5557e7665200,0x5557e7631b00> 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9dd5c70 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5ce0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630300 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5cf8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 finishing 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630300 onodes 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630300 done 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631b00 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630300 [] 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5d50 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5dc0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693200 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5d50) get_ref 0x0~2000 Blob(0x5557e9dd5d50 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5dc0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 finishing 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9dd5d50 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5dc0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693200 onodes 0x5557e9eb4a00 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5dd8 in 0x5557e7072000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56be8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b598c0 space 0x5557e9b56be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be39e8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e9be39e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5e30 blob([0x234000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5ea0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d375f8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b599e0 space 0x5557e9d375f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5e30) get_ref 0x0~1000 Blob(0x5557e9dd5e30 blob([0x234000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5ea0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37668 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b59a70 space 0x5557e9d37668 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.blob(0x5557e9dd5e30) get_ref init 0x1000, 1000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d377b8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7b00 space 0x5557e9d377b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~1000: 0x0~1000 Blob(0x5557e9dd5e30 blob([0x234000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9dd5ea0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37908 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7a70 space 0x5557e9d37908 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9dd5eb8 in 0x5557e7072000) _discard 0x0~1000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37978 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9d37978 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=23, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e74f1c00 con 0x5557e79ca800 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37a58 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9d37a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37f28 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b59950 space 0x5557e9d37f28 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb7048 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9eb7048 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb7128 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e79ab9e0 space 0x5557e9eb7128 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb7208 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9eb7208 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb72e8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9eb72e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb73c8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9eb73c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x21000 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb74a8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7c20 space 0x5557e9eb74a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb7588 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7cb0 space 0x5557e9eb7588 0x0~1000 writing nocache) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.extentmap(0x5557ea010388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.710+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 = 0 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693200 done 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693200 [] 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.710+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.710+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665200 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 finishing 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665200 onodes 0x5557e76cac80 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665200 done 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.710+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665200 [] 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 kv_submitted 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631b00 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631b00 finishing 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631b00 onodes 0x5557e9b95400 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631b00 done 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631b00 [] 2019-12-07T19:33:58.710+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.710+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5b800 cost 208790 (18 ios * 4000 + 136790 bytes) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5b800 onodes 0x5557ea010280 shared_blobs 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.extentmap(0x5557ea010388) update 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore.extentmap(0x5557ea010388) update inline shard 323 bytes from 17 extents 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:fc3a86a2:::smithi13140796-33:head# is 742 (413 bytes onode + 2 bytes spanning blobs + 327 bytes inline extents) 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5b800 allocated 0x[214000~21000] released 0x[] 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 freelist allocate 0x214000~21000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 prepare 2019-12-07T19:33:58.710+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9d5b800 2019-12-07T19:33:58.710+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.710+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=13, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e9c77000 con 0x5557e79cbc00 2019-12-07T19:33:58.710+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=14, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e76b1200 con 0x5557e79cbc00 2019-12-07T19:33:58.710+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=15, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e758da00 con 0x5557e79cbc00 2019-12-07T19:33:58.710+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.711+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.711+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d84b00 finish 2019-12-07T19:33:58.711+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.711+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.711+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.711+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=45, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e9d88400 con 0x5557e79cac00 2019-12-07T19:33:58.711+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=24, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e9ef3c00 con 0x5557e79ca800 2019-12-07T19:33:58.711+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=46, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e9ef3a00 con 0x5557e79cac00 2019-12-07T19:33:58.711+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 aio_wait 2019-12-07T19:33:58.713+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5b800 2019-12-07T19:33:58.713+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 io_done 2019-12-07T19:33:58.713+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 aio_wait 2019-12-07T19:33:58.713+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10300 2019-12-07T19:33:58.713+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 io_done 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.713+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1454 < min_flush_size 524288 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea010280 had 1 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.713+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2921 < min_flush_size 524288 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb5680 had 1 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.713+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2944 < min_flush_size 524288 2019-12-07T19:33:58.713+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2944 < min_flush_size 524288 2019-12-07T19:33:58.713+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.713+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x41fc1~b80 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.713+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x41fc1 0x41fc1~b80 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x11fc1 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xfc1 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.713+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xb41 and padding block with 0x4bf 2019-12-07T19:33:58.713+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 83 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 141236+0+0 (crc 0 0 0) 0x5557e9c79800 con 0x5557e79cac00 2019-12-07T19:33:58.713+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85ce0 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.713+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] add_log_entry 21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7a92600 seq 16 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.7s1_head 0x5557e76a92c0) get_onode oid 1#1:e7f24557:::smithi13140796-34:head# key 0x818000000000000001e7f245'W!smithi13140796-34!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.713+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1115 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# = 0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 2 keys 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 2 keys = 0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 0x0~22000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:e7f24557:::smithi13140796-34:head# 0x0~22000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~22000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a92600 17 blobs 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate 0x22000/1000,22000,0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x235000~22000/1000,22000,0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x235000~22000] 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5acb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2a80 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5acb0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9be2a80) get_ref 0x0~2000 Blob(0x5557e9be2a80 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5acb0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9be2a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9be2a80 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5acb0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5acc8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b70770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b70770 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36a80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b70770) get_ref 0x0~2000 Blob(0x5557e9b70770 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36a80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b70770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9b70770 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36a80 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36a98 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7b9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7b9d0 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36a10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b9d0) get_ref 0x0~2000 Blob(0x5557e9b7b9d0 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36a10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b9d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9b7b9d0 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36a10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36a28 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36af0 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36b60 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36af0) get_ref 0x0~2000 Blob(0x5557e9c36af0 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36b60 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36af0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9c36af0 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36b60 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36b78 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36bd0 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36c40 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36bd0) get_ref 0x0~2000 Blob(0x5557e9c36bd0 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36c40 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9c36bd0 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36c40 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36c58 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36cb0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36d20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36cb0) get_ref 0x0~2000 Blob(0x5557e9c36cb0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36d20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9c36cb0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36d20 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36d38 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36d90 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36e00 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36d90) get_ref 0x0~2000 Blob(0x5557e9c36d90 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36e00 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9c36d90 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36e00 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36e18 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36e70 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36ee0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36e70) get_ref 0x0~2000 Blob(0x5557e9c36e70 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36ee0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9c36e70 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36ee0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36ef8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36f50 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36fc0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36f50) get_ref 0x0~2000 Blob(0x5557e9c36f50 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36fc0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9c36f50 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36fc0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36fd8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b7a0 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b810 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b7a0) get_ref 0x0~2000 Blob(0x5557e9e5b7a0 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b810 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b7a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9e5b7a0 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b810 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b828 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b880 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b8f0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b880) get_ref 0x0~2000 Blob(0x5557e9e5b880 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b8f0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9e5b880 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b8f0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b908 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b960 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b9d0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b960) get_ref 0x0~2000 Blob(0x5557e9e5b960 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b9d0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9e5b960 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b9d0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b9e8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5ba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5ba40 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bab0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5ba40) get_ref 0x0~2000 Blob(0x5557e9e5ba40 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bab0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5ba40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9e5ba40 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bab0 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bac8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5bb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5bb20 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bb90 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bb20) get_ref 0x0~2000 Blob(0x5557e9e5bb20 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bb90 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bb20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9e5bb20 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bb90 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bba8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5bc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5bc00 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bc70 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bc00) get_ref 0x0~2000 Blob(0x5557e9e5bc00 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bc70 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bc00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9e5bc00 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bc70 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bc88 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5bce0 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bd50 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bce0) get_ref 0x0~2000 Blob(0x5557e9e5bce0 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bd50 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9e5bce0 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bd50 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bd68 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5bdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5be30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5bdc0 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5be30 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bdc0) get_ref 0x0~2000 Blob(0x5557e9e5bdc0 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5be30 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bdc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9e5bdc0 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5be30 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5be48 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x22000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 0x0~22000 = 0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92600 cost 212886 (18 ios * 4000 + 140886 bytes) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92600 onodes 0x5557e9b95180 shared_blobs 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95288) update 1#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95288) update inline shard 326 bytes from 17 extents 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:e7f24557:::smithi13140796-34:head# is 745 (413 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92600 allocated 0x[235000~22000] released 0x[] 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 freelist allocate 0x235000~22000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 prepare 2019-12-07T19:33:58.713+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a92600 2019-12-07T19:33:58.714+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.714+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 finish 2019-12-07T19:33:58.714+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.715+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 84 ==== MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9d59b80 con 0x5557e79cac00 2019-12-07T19:33:58.715+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.715+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000054 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.715+0000 7f2b76cdf700 25 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.715+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~1000 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x19000 (102400) 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9c2b908 in 0x5557e6269880) _discard 0x0~1000 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 _add level 1 near 0 on buffer(0x5557e9f00630 space 0x5557e9c2b908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~1000 = 4096 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] get_hash_info: found on disk, size 102400 2019-12-07T19:33:58.715+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:33:58.715+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=50, attrs_read=0)) v2 -- 0x5557ea010000 con 0x5557e79cac00 2019-12-07T19:33:58.715+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.715+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x42b41 2019-12-07T19:33:58.715+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.715+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.715+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.715+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.715+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.002296876s (0.000057347s flush + 0.002239529s kv commit) 2019-12-07T19:33:58.715+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5b800,0x5557e9b10300> 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.715+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 kv_submitted 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5b800 2019-12-07T19:33:58.715+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b800 finishing 2019-12-07T19:33:58.715+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 aio_wait 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5b800 onodes 0x5557ea010280 2019-12-07T19:33:58.715+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92600 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2be48 in 0x5557e7072000) _finish_write discard buffer(0x5557e9b58630 space 0x5557e9c2be48 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37c18 in 0x5557e7072000) _finish_write discard buffer(0x5557e9e45cb0 space 0x5557e9d37c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 io_done 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5278 in 0x5557e7072000) _finish_write discard buffer(0x5557e9e45d40 space 0x5557e9dd5278 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5358 in 0x5557e7072000) _finish_write discard buffer(0x5557e9e45dd0 space 0x5557e9dd5358 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5438 in 0x5557e7072000) _finish_write discard buffer(0x5557e9cd70e0 space 0x5557e9dd5438 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5518 in 0x5557e7072000) _finish_write discard buffer(0x5557e9e45e60 space 0x5557e9dd5518 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd55f8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9e45ef0 space 0x5557e9dd55f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd56d8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f00000 space 0x5557e9dd56d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd57b8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f00090 space 0x5557e9dd57b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5898 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f00120 space 0x5557e9dd5898 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5978 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f001b0 space 0x5557e9dd5978 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5a58 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f00240 space 0x5557e9dd5a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5b38 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f002d0 space 0x5557e9dd5b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5c18 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f00360 space 0x5557e9dd5c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5cf8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f003f0 space 0x5557e9dd5cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.715+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5dd8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f00480 space 0x5557e9dd5dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5eb8 in 0x5557e7072000) _finish_write discard buffer(0x5557e9f00510 space 0x5557e9dd5eb8 0x0~1000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5b800 done 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.715+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5b800 [] 2019-12-07T19:33:58.715+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.715+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 kv_submitted 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10300 2019-12-07T19:33:58.715+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 finishing 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10300 onodes 0x5557e9eb5680 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2bd68 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e44000 space 0x5557e9c2bd68 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37198 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e441b0 space 0x5557e9c37198 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37278 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e44240 space 0x5557e9c37278 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=26, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e76b1e00 con 0x5557e79ca800 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37358 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e442d0 space 0x5557e9c37358 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37438 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd6360 space 0x5557e9c37438 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c37518 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e44360 space 0x5557e9c37518 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd4088 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e443f0 space 0x5557e9dd4088 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd4168 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e45680 space 0x5557e9dd4168 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd4248 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e45710 space 0x5557e9dd4248 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd4328 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e457a0 space 0x5557e9dd4328 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd4408 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e45830 space 0x5557e9dd4408 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd44e8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e458c0 space 0x5557e9dd44e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd45c8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e45950 space 0x5557e9dd45c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd46a8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e459e0 space 0x5557e9dd46a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd4788 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e45a70 space 0x5557e9dd4788 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd4868 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e45b00 space 0x5557e9dd4868 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10300 done 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.715+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10300 [] 2019-12-07T19:33:58.715+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.715+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.715+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=48, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e9c76000 con 0x5557e79cac00 2019-12-07T19:33:58.715+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.716+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 44 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9c7a000 con 0x5557e79cbc00 2019-12-07T19:33:58.716+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5600 prio 127 cost 0 latency 0.000029 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.716+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c5600 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.3s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:86, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'5], crt=21'3) [21'6 (21'4) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:86 2019-12-07T19:33:58.693762+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'3 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'4) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:86 2019-12-07T19:33:58.693762+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] rollforward: entry=21'4 (0'0) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:85 2019-12-07T19:33:58.693730+0000 0 ObjectCleanRegions clean_offsets: [475136~18446744073709076479], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.716+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7630900 seq 12 2019-12-07T19:33:58.716+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 3 keys 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 3 keys = 0 2019-12-07T19:33:58.716+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.716+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630900 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630900 onodes 0x5557e9eb4000 shared_blobs 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9eb4108) update 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:c8ce3553:::smithi13140796-29:head# is 776 (485 bytes onode + 2 bytes spanning blobs + 289 bytes inline extents) 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630900 prepare 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630900 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630900 io_done 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.716+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c5600 finish 2019-12-07T19:33:58.716+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.716+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 54 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c7a800 con 0x5557e79ca800 2019-12-07T19:33:58.716+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.716+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:91, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.716+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 45 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c7b800 con 0x5557e79cbc00 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'7], crt=21'6) [] 2019-12-07T19:33:58.716+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e76958c0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.716+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (0'0) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:91 2019-12-07T19:33:58.699920+0000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.716+0000 7f2b774e0700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9cc4900 seq 13 2019-12-07T19:33:58.716+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e76958c0 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4900 onodes shared_blobs 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 prepare 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4900 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 io_done 2019-12-07T19:33:58.716+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c58c0 finish 2019-12-07T19:33:58.716+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.716+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7664900 seq 12 2019-12-07T19:33:58.716+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664900 onodes shared_blobs 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 prepare 2019-12-07T19:33:58.716+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664900 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 io_done 2019-12-07T19:33:58.716+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e76958c0 finish 2019-12-07T19:33:58.717+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.717+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 46 ==== MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557ea010000 con 0x5557e79cbc00 2019-12-07T19:33:58.717+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695b80 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.717+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.717+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e7695b80 prio 127 cost 0 latency 0.000061 MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.717+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.717+0000 7f2b74cdb700 25 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.717+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1000 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x1c000 (114688) 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b70690 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2ba40 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b70690 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2ba40 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.717+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.717+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 55 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9ba0000 con 0x5557e79ca800 2019-12-07T19:33:58.717+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695600 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.717+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.717+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 85 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=51, reqid=client.4540.0:94, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c79000 con 0x5557e79cac00 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7695600 prio 127 cost 0 latency 0.000055 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.717+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=51, reqid=client.4540.0:94, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.717+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=51, reqid=client.4540.0:94, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:92, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.717+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.717+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=51, reqid=client.4540.0:94, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.717+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.717+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 86 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=52, reqid=client.4540.0:95, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9d80000 con 0x5557e79cac00 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:92 2019-12-07T19:33:58.699945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.717+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=51, reqid=client.4540.0:94, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.717+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d851e0 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=52, reqid=client.4540.0:95, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.717+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=52, reqid=client.4540.0:95, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'7) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:92 2019-12-07T19:33:58.699945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.717+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000053 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=51, reqid=client.4540.0:94, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.717+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.717+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=52, reqid=client.4540.0:95, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.717+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=51, reqid=client.4540.0:94, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.717+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.717+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'7], crt=21'6) [] 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7693800 seq 14 2019-12-07T19:33:58.717+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (0'0) modify 1:10779340:::smithi13140796-32:head by client.4540.0:94 2019-12-07T19:33:58.703907+0000 0 ObjectCleanRegions clean_offsets: [524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.717+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 3 keys 2019-12-07T19:33:58.717+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 3 keys = 0 2019-12-07T19:33:58.717+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.717+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.717+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.717+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.717+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693800 onodes 0x5557e9eb4a00 shared_blobs 2019-12-07T19:33:58.717+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb4b08) update 1#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:33:58.717+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9cc4f00 seq 16 2019-12-07T19:33:58.717+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.717+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.717+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.717+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:6ad15c4c:::smithi13140796-31:head# is 795 (485 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:33:58.717+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.717+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.717+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4f00 onodes shared_blobs 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 prepare 2019-12-07T19:33:58.717+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693800 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 io_done 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.717+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4f00 prepare 2019-12-07T19:33:58.717+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7695600 finish 2019-12-07T19:33:58.717+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4f00 2019-12-07T19:33:58.717+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4f00 io_done 2019-12-07T19:33:58.717+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.717+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85340 finish 2019-12-07T19:33:58.717+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.717+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.717+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=52, reqid=client.4540.0:95, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.717+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d851e0 prio 127 cost 0 latency 0.000164 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=52, reqid=client.4540.0:95, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.717+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=52, reqid=client.4540.0:95, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.717+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:10779340:::smithi13140796-32:head by client.4540.0:95 2019-12-07T19:33:58.703940+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.717+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'7) modify 1:10779340:::smithi13140796-32:head by client.4540.0:95 2019-12-07T19:33:58.703940+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1457 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7a93800 seq 17 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 3 keys 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 3 keys = 0 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93800 onodes 0x5557e9eb5680 shared_blobs 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9eb5788) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 798 (485 bytes onode + 2 bytes spanning blobs + 311 bytes inline extents) 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 prepare 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93800 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 io_done 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95180 had 1 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1480 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1480 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 10 osd.6 21 dequeue_op 0x5557e9d851e0 finish 2019-12-07T19:33:58.718+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x42b41~5c8 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x42b41 0x42b41~5c8 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x12b41 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xb41 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x109 and padding block with 0xef7 2019-12-07T19:33:58.718+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b70690 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2ba40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.718+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9c2ba58 in 0x5557e6269c00) _discard 0x0~1000 2019-12-07T19:33:58.718+0000 7f2b74cdb700 20 _add level 1 near 0 on buffer(0x5557e9e450e0 space 0x5557e9c2ba58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.718+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 0x0~1000 = 4096 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x43109 2019-12-07T19:33:58.718+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.718+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:33:58.718+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.002516177s (0.002306835s flush + 0.000209342s kv commit) 2019-12-07T19:33:58.718+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.718+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 114688 2019-12-07T19:33:58.718+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:33:58.718+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key = 0 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1816 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b74cdb700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 21/18 ECSubReadReply(tid=18, attrs_read=0)) v2 -- 0x5557e76ca500 con 0x5557e79cbc00 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4000 had 1 2019-12-07T19:33:58.718+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e7695b80 finish 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2930 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92600> 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92600 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 2 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 finishing 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92600 onodes 0x5557e9b95180 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36a28 in 0x5557e6269880) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9c36a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4044 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36a98 in 0x5557e6269880) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9c36a98 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36b78 in 0x5557e6269880) _finish_write discard buffer(0x5557e9e44480 space 0x5557e9c36b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36c58 in 0x5557e6269880) _finish_write discard buffer(0x5557e9e44510 space 0x5557e9c36c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36d38 in 0x5557e6269880) _finish_write discard buffer(0x5557e9e445a0 space 0x5557e9c36d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5879 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36e18 in 0x5557e6269880) _finish_write discard buffer(0x5557e9e44630 space 0x5557e9c36e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36ef8 in 0x5557e6269880) _finish_write discard buffer(0x5557e9e44090 space 0x5557e9c36ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36fd8 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aaea0 space 0x5557e9c36fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5acc8 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aa750 space 0x5557e9e5acc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b828 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aac60 space 0x5557e9e5b828 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b908 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aa120 space 0x5557e9e5b908 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=49, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e9b36400 con 0x5557e79cac00 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b9e8 in 0x5557e6269880) _finish_write discard buffer(0x5557e79aa360 space 0x5557e9e5b9e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4a00 had 1 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5bac8 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd62d0 space 0x5557e9e5bac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5bba8 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e9e5bba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5bc88 in 0x5557e6269880) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9e5bc88 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6993 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5bd68 in 0x5557e6269880) _finish_write discard buffer(0x5557e9f00750 space 0x5557e9e5bd68 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5be48 in 0x5557e6269880) _finish_write discard buffer(0x5557e9f007e0 space 0x5557e9e5be48 0x0~2000 writing nocache) 2019-12-07T19:33:58.718+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 2 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8831 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92600 done 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92600 [] 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb5680 had 1 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8854 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8854 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x43109~2296 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x43109 0x43109~2296 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x13109 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x109 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x39f and padding block with 0xc61 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4539f 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.718+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 6 cleaned 0 in 0.000402733s (0.000000074s flush + 0.000402659s kv commit) 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.718+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630900,0x5557e9cc4900,0x5557e7664900,0x5557e7693800,0x5557e9cc4f00,0x5557e7a93800> 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630900 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630900 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630900 finishing 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630900 onodes 0x5557e9eb4000 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630900 done 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630900 [] 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4900 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 finishing 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4900 onodes 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4900 done 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693800 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4900 [] 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664900 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 finishing 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664900 onodes 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664900 done 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664900 [] 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693800 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 finishing 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693800 onodes 0x5557e9eb4a00 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693800 done 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693800 [] 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4f00 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4f00 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4f00 finishing 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4f00 onodes 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4f00 done 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93800 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4f00 [] 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 kv_submitted 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93800 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 finishing 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93800 onodes 0x5557e9eb5680 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93800 done 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93800 [] 2019-12-07T19:33:58.718+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.718+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.718+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=27, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e9d89000 con 0x5557e79ca800 2019-12-07T19:33:58.718+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=28, last_complete=21'8, committed=1, applied=1)) v2 -- 0x5557e7a9b800 con 0x5557e79ca800 2019-12-07T19:33:58.718+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=16, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e9b37000 con 0x5557e79cbc00 2019-12-07T19:33:58.718+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=17, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e9c76000 con 0x5557e79cbc00 2019-12-07T19:33:58.718+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.719+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 31 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:103, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 145332+0+0 (crc 0 0 0) 0x5557e9c78800 con 0x5557e7519800 2019-12-07T19:33:58.719+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695340 prio 127 cost 0 latency 0.000029 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:103, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.719+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.719+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:103, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:103, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.719+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=51, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e9c76600 con 0x5557e79cac00 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:103, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 osd.6 21 dequeue_op 0x5557e7695340 prio 127 cost 0 latency 0.000279 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:103, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.719+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=52, last_complete=21'8, committed=1, applied=1)) v2 -- 0x5557e9ef3400 con 0x5557e79cac00 2019-12-07T19:33:58.719+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:103, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:103 2019-12-07T19:33:58.715018+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:103 2019-12-07T19:33:58.715018+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.719+0000 7f2b77ce1700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9cc4300 seq 13 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.5s1_head 0x5557e6281c20) get_onode oid 1#1:a7686634:::smithi13140796-35:head# key 0x818000000000000001a76866'4!smithi13140796-35!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.719+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1116 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# = 0 2019-12-07T19:33:58.719+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 2 keys 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 2 keys = 0 2019-12-07T19:33:58.719+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.719+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 0x0~23000 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:a7686634:::smithi13140796-35:head# 0x0~23000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.719+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~23000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.719+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9cc4300 18 blobs 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 fbmap_alloc 0x5557e6282e00 allocate 0x23000/1000,23000,0 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x257000~23000/1000,23000,0 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x257000~23000] 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5abd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5abd0 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a930 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5abd0) get_ref 0x0~2000 Blob(0x5557e9e5abd0 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b7a930 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5abd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9e5abd0 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a930 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9b7a948 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5ab60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5aaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5ab60 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5aaf0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5ab60) get_ref 0x0~2000 Blob(0x5557e9e5ab60 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5aaf0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5ab60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9e5ab60 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5aaf0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9e5ab08 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5aa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5aa10 blob([0x25b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a9a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5aa10) get_ref 0x0~2000 Blob(0x5557e9e5aa10 blob([0x25b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a9a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5aa10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9e5aa10 blob([0x25b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5a9a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9e5a9b8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5a930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5a930 blob([0x25d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a8c0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5a930) get_ref 0x0~2000 Blob(0x5557e9e5a930 blob([0x25d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a8c0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5a930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9e5a930 blob([0x25d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5a8c0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9e5a8d8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5a850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5a850 blob([0x25f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a7e0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5a850) get_ref 0x0~2000 Blob(0x5557e9e5a850 blob([0x25f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a7e0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5a850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9e5a850 blob([0x25f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5a7e0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9e5a7f8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5a770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5a770 blob([0x261000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a700 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5a770) get_ref 0x0~2000 Blob(0x5557e9e5a770 blob([0x261000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5a700 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5a770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9e5a770 blob([0x261000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5a700 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9e5a718 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5a690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5a690 blob([0x263000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bdc0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5a690) get_ref 0x0~2000 Blob(0x5557e9e5a690 blob([0x263000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bdc0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5a690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9e5a690 blob([0x263000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2bdc0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9c2bdd8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0a930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0a9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0a930 blob([0x265000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0a9a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0a930) get_ref 0x0~2000 Blob(0x5557e9f0a930 blob([0x265000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0a9a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0a930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9f0a930 blob([0x265000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0a9a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0a9b8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0aa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0aa80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0aa10 blob([0x267000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0aa80 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0aa10) get_ref 0x0~2000 Blob(0x5557e9f0aa10 blob([0x267000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0aa80 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0aa10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9f0aa10 blob([0x267000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0aa80 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0aa98 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0aaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ab60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0aaf0 blob([0x269000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ab60 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0aaf0) get_ref 0x0~2000 Blob(0x5557e9f0aaf0 blob([0x269000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ab60 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0aaf0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9f0aaf0 blob([0x269000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0ab60 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0ab78 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0abd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ac40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0abd0 blob([0x26b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ac40 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0abd0) get_ref 0x0~2000 Blob(0x5557e9f0abd0 blob([0x26b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ac40 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0abd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9f0abd0 blob([0x26b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0ac40 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0ac58 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0acb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ad20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0acb0 blob([0x26d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ad20 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0acb0) get_ref 0x0~2000 Blob(0x5557e9f0acb0 blob([0x26d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ad20 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0acb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9f0acb0 blob([0x26d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0ad20 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0ad38 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0ad90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ae00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0ad90 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ae00 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0ad90) get_ref 0x0~2000 Blob(0x5557e9f0ad90 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0ae00 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0ad90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9f0ad90 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0ae00 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0ae18 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0ae70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0aee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0ae70 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0aee0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0ae70) get_ref 0x0~2000 Blob(0x5557e9f0ae70 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0aee0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0ae70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9f0ae70 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0aee0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0aef8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5ad90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ad20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5ad90 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ad20 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5ad90) get_ref 0x0~2000 Blob(0x5557e9e5ad90 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ad20 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9e5ad90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9e5ad90 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5ad20 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9e5ad38 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0af50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0afc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0af50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0afc0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0af50) get_ref 0x0~2000 Blob(0x5557e9f0af50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0afc0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0af50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9f0af50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0afc0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0afd8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0b030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0b0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0b030 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0b0a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0b030) get_ref 0x0~2000 Blob(0x5557e9f0b030 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0b0a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0b030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9f0b030 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0b0a0 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0b0b8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0b110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0b180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0b110 blob([0x279000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0b180 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0b110) get_ref 0x0~1000 Blob(0x5557e9f0b110 blob([0x279000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0b180 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.blob(0x5557e9f0b110) get_ref init 0x1000, 1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~1000: 0x0~1000 Blob(0x5557e9f0b110 blob([0x279000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9f0b180 sbid 0x0)) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.BufferSpace(0x5557e9f0b198 in 0x5557e6269180) _discard 0x0~1000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x23000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557ea010888) dirty_range mark inline shard dirty 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 0x0~23000 = 0 2019-12-07T19:33:58.720+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.720+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4300 cost 220982 (19 ios * 4000 + 144982 bytes) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4300 onodes 0x5557ea010780 shared_blobs 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557ea010888) update 1#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557ea010888) update inline shard 342 bytes from 18 extents 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:a7686634:::smithi13140796-35:head# is 761 (413 bytes onode + 2 bytes spanning blobs + 346 bytes inline extents) 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4300 allocated 0x[257000~23000] released 0x[] 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 freelist allocate 0x257000~23000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 prepare 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9cc4300 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.720+0000 7f2b77ce1700 10 osd.6 21 dequeue_op 0x5557e7695340 finish 2019-12-07T19:33:58.720+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.720+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 aio_wait 2019-12-07T19:33:58.720+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4300 2019-12-07T19:33:58.720+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 io_done 2019-12-07T19:33:58.720+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.720+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.720+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1473 < min_flush_size 524288 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea010780 had 1 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1496 < min_flush_size 524288 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1496 < min_flush_size 524288 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4539f~5d8 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4539f 0x4539f~5d8 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1539f 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x39f 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x977 and padding block with 0x689 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x45977 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.721+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000149407s (0.000042560s flush + 0.000106847s kv commit) 2019-12-07T19:33:58.721+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4300> 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.721+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 kv_submitted 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4300 2019-12-07T19:33:58.721+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4300 finishing 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4300 onodes 0x5557ea010780 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b7a948 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9b7a948 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2bdd8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e9c2bdd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5a718 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd6360 space 0x5557e9e5a718 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5a7f8 in 0x5557e6269180) _finish_write discard buffer(0x5557e79ab9e0 space 0x5557e9e5a7f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5a8d8 in 0x5557e6269180) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9e5a8d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5a9b8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b59a70 space 0x5557e9e5a9b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5ab08 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b599e0 space 0x5557e9e5ab08 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5ad38 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9e5ad38 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0a9b8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd7b00 space 0x5557e9f0a9b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0aa98 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd7a70 space 0x5557e9f0aa98 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0ab78 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b59950 space 0x5557e9f0ab78 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0ac58 in 0x5557e6269180) _finish_write discard buffer(0x5557e9b58630 space 0x5557e9f0ac58 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0ad38 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9f0ad38 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0ae18 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9f0ae18 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0aef8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9f0aef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0afd8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9f0afd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0b0b8 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd7c20 space 0x5557e9f0b0b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0b198 in 0x5557e6269180) _finish_write discard buffer(0x5557e9cd7cb0 space 0x5557e9f0b198 0x0~1000 writing nocache) 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4300 done 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.721+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4300 [] 2019-12-07T19:33:58.721+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.721+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.721+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=13, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e9ef2600 con 0x5557e7519800 2019-12-07T19:33:58.721+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 87 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=54, reqid=client.4540.0:95, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d80800 con 0x5557e79cac00 2019-12-07T19:33:58.723+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=54, reqid=client.4540.0:95, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.723+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=54, reqid=client.4540.0:95, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.723+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=54, reqid=client.4540.0:95, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.723+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=54, reqid=client.4540.0:95, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.723+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=54, reqid=client.4540.0:95, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.723+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=54, reqid=client.4540.0:95, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.723+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'8], crt=21'7) [] 2019-12-07T19:33:58.723+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] rollforward: entry=21'8 (21'7) modify 1:10779340:::smithi13140796-32:head by client.4540.0:95 2019-12-07T19:33:58.703940+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.723+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.723+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.723+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.723+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.723+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.723+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9cc4c00 seq 18 2019-12-07T19:33:58.723+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.723+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.723+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.723+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.723+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.723+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.723+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4c00 onodes shared_blobs 2019-12-07T19:33:58.723+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.723+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4c00 prepare 2019-12-07T19:33:58.723+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4c00 2019-12-07T19:33:58.723+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4c00 io_done 2019-12-07T19:33:58.723+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85760 finish 2019-12-07T19:33:58.723+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x45977~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x45977 0x45977~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x15977 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x977 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xde8 and padding block with 0x218 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x45de8 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000090990s (0.000000075s flush + 0.000090915s kv commit) 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.723+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.723+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4c00> 2019-12-07T19:33:58.723+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.723+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4c00 kv_submitted 2019-12-07T19:33:58.723+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4c00 2019-12-07T19:33:58.723+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4c00 finishing 2019-12-07T19:33:58.723+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4c00 onodes 2019-12-07T19:33:58.723+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4c00 done 2019-12-07T19:33:58.723+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.723+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4c00 [] 2019-12-07T19:33:58.723+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.723+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.723+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 88 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e3000 con 0x5557e79cac00 2019-12-07T19:33:58.723+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85a20 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.723+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85a20 prio 127 cost 0 latency 0.000043 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'7], crt=21'5) [] 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.723+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7a92c00 seq 17 2019-12-07T19:33:58.723+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes shared_blobs 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:58.723+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:58.723+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=54, last_complete=21'8, committed=1, applied=1)) v2 -- 0x5557e9d88400 con 0x5557e79cac00 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.723+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85a20 finish 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.723+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x45de8~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.723+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x45de8 0x45de8~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x15de8 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xde8 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.723+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x259 and padding block with 0xda7 2019-12-07T19:33:58.724+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x46259 2019-12-07T19:33:58.724+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.724+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.724+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.724+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.724+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000126731s (0.000000050s flush + 0.000126681s kv commit) 2019-12-07T19:33:58.724+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.724+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.724+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92c00> 2019-12-07T19:33:58.724+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.724+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:58.724+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:58.724+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:58.724+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 2019-12-07T19:33:58.724+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:58.724+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.724+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:58.724+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.724+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.724+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.724+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=56, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e9ef3a00 con 0x5557e79cac00 2019-12-07T19:33:58.724+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.724+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 56 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e2800 con 0x5557e79ca800 2019-12-07T19:33:58.724+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85080 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.724+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.724+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.724+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.724+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.724+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.724+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85080 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.724+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.724+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.724+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:97 2019-12-07T19:33:58.707663+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.724+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.724+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.724+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.724+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7693b00 seq 14 2019-12-07T19:33:58.725+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693b00 onodes shared_blobs 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 prepare 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693b00 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 io_done 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85080 finish 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x46259~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x46259 0x46259~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x16259 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x259 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x6ca and padding block with 0x936 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x466ca 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000123585s (0.000000058s flush + 0.000123527s kv commit) 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693b00> 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.725+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 kv_submitted 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693b00 2019-12-07T19:33:58.725+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 finishing 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693b00 onodes 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693b00 done 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.725+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693b00 [] 2019-12-07T19:33:58.725+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.725+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=29, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e9ef3800 con 0x5557e79ca800 2019-12-07T19:33:58.725+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.725+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 57 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e763e800 con 0x5557e79ca800 2019-12-07T19:33:58.725+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.725+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:98 2019-12-07T19:33:58.707684+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:98 2019-12-07T19:33:58.707684+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7046f00 seq 15 2019-12-07T19:33:58.725+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 3 keys 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# 3 keys = 0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.fs5_head 5#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes 0x5557ea010280 shared_blobs 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557ea010388) update 5#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:fc3a86a2:::smithi13140796-33:head# is 814 (485 bytes onode + 2 bytes spanning blobs + 327 bytes inline extents) 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1854 < min_flush_size 524288 2019-12-07T19:33:58.725+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d85600 finish 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea010280 had 1 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1877 < min_flush_size 524288 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1877 < min_flush_size 524288 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x466ca~755 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x466ca 0x466ca~755 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x166ca 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x6ca 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe1f and padding block with 0x1e1 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x46e1f 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.725+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000130771s (0.000000070s flush + 0.000130701s kv commit) 2019-12-07T19:33:58.725+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.725+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:58.725+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 0x5557ea010280 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.725+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:58.725+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.725+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.725+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.725+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=30, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e9ef3600 con 0x5557e79ca800 2019-12-07T19:33:58.725+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 89 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9d81800 con 0x5557e79cac00 2019-12-07T19:33:58.726+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d854a0 prio 127 cost 0 latency 0.000039 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.726+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d854a0 prio 127 cost 0 latency 0.000087 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.726+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7693500 seq 18 2019-12-07T19:33:58.726+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 3 keys 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 3 keys = 0 2019-12-07T19:33:58.726+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.726+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693500 onodes 0x5557e9b95180 shared_blobs 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95288) update 1#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:e7f24557:::smithi13140796-34:head# is 817 (485 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 prepare 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693500 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 io_done 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.726+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d854a0 finish 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1857 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95180 had 1 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1880 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1880 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x46e1f~758 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x46e1f 0x46e1f~758 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x16e1f 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe1f 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x577 and padding block with 0xa89 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x47577 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.726+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000131089s (0.000000094s flush + 0.000130995s kv commit) 2019-12-07T19:33:58.726+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.726+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.726+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693500> 2019-12-07T19:33:58.726+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.726+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 kv_submitted 2019-12-07T19:33:58.726+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693500 2019-12-07T19:33:58.726+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 finishing 2019-12-07T19:33:58.726+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693500 onodes 0x5557e9b95180 2019-12-07T19:33:58.726+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693500 done 2019-12-07T19:33:58.726+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.726+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693500 [] 2019-12-07T19:33:58.726+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.726+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.726+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=57, last_complete=21'8, committed=1, applied=1)) v2 -- 0x5557e9ef3800 con 0x5557e79cac00 2019-12-07T19:33:58.726+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.727+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 58 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d83800 con 0x5557e79ca800 2019-12-07T19:33:58.727+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.727+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000044 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=31, reqid=client.4540.0:92, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'8], crt=21'7) [] 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] rollforward: entry=21'8 (21'7) modify 1:6ad15c4c:::smithi13140796-31:head by client.4540.0:92 2019-12-07T19:33:58.699945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.727+0000 7f2b774e0700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9cc4600 seq 15 2019-12-07T19:33:58.727+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4600 onodes shared_blobs 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 prepare 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4600 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 io_done 2019-12-07T19:33:58.727+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e9d84160 finish 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x47577~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x47577 0x47577~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x17577 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x577 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x9e8 and padding block with 0x618 2019-12-07T19:33:58.727+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 59 ==== MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e76ca500 con 0x5557e79ca800 2019-12-07T19:33:58.727+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d849a0 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.727+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d849a0 prio 127 cost 0 latency 0.000047 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=32, to_read={1:6ad15c4c:::smithi13140796-31:head=0,4096,0}, subchunks={1:6ad15c4c:::smithi13140796-31:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.727+0000 7f2b734d8700 25 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.727+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 0x0~1000 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x1f000 (126976) 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9be3b20 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b56bd0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9be3b20 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b56bd0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x479e8 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.727+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.727+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000130743s (0.000000079s flush + 0.000130664s kv commit) 2019-12-07T19:33:58.727+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4600> 2019-12-07T19:33:58.727+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.727+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.727+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 kv_submitted 2019-12-07T19:33:58.727+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4600 2019-12-07T19:33:58.727+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4600 finishing 2019-12-07T19:33:58.727+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4600 onodes 2019-12-07T19:33:58.727+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4600 done 2019-12-07T19:33:58.727+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.727+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4600 [] 2019-12-07T19:33:58.727+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.727+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9be3b20 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b56bd0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9b56be8 in 0x5557e6268fc0) _discard 0x0~1000 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 _add level 1 near 0 on buffer(0x5557e9cd7d40 space 0x5557e9b56be8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 0x0~1000 = 4096 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:6ad15c4c:::smithi13140796-31:head 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 126976 2019-12-07T19:33:58.727+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# hinfo_key 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:6ad15c4c:::smithi13140796-31:head# hinfo_key = 0 2019-12-07T19:33:58.727+0000 7f2b734d8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=32, attrs_read=0)) v2 -- 0x5557e9bc0a00 con 0x5557e79ca800 2019-12-07T19:33:58.727+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d849a0 finish 2019-12-07T19:33:58.727+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.727+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=31, last_complete=21'8, committed=1, applied=1)) v2 -- 0x5557e76b1e00 con 0x5557e79ca800 2019-12-07T19:33:58.727+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 32 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9ba0000 con 0x5557e7519800 2019-12-07T19:33:58.729+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 47 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d83800 con 0x5557e79cbc00 2019-12-07T19:33:58.729+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c4840 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.729+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.729+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.729+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4840 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:103, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:103 2019-12-07T19:33:58.715018+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.729+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9b11b00 seq 14 2019-12-07T19:33:58.729+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11b00 onodes shared_blobs 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 prepare 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11b00 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 io_done 2019-12-07T19:33:58.729+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c4840 finish 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 127 cost 0 latency 0.000191 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x479e8~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'6], crt=21'4) [] 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x479e8 0x479e8~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x179e8 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x9e8 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe59 and padding block with 0x1a7 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'5 (21'3) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:80 2019-12-07T19:33:58.687362+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'4) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:86 2019-12-07T19:33:58.693762+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.729+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7630000 seq 13 2019-12-07T19:33:58.729+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630000 onodes shared_blobs 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 prepare 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630000 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 io_done 2019-12-07T19:33:58.729+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x47e59 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000151468s (0.000000077s flush + 0.000151391s kv commit) 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 423 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1148 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1148 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x47e59~47c to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x47e59 0x47e59~47c to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x17e59 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe59 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x2d5 and padding block with 0xd2b 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11b00> 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.729+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 kv_submitted 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11b00 2019-12-07T19:33:58.729+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 finishing 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11b00 onodes 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11b00 done 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.729+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11b00 [] 2019-12-07T19:33:58.729+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.729+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=16, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e7a9b400 con 0x5557e7519800 2019-12-07T19:33:58.729+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x482d5 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.729+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000125881s (0.000000058s flush + 0.000125823s kv commit) 2019-12-07T19:33:58.729+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630000> 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.729+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 kv_submitted 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630000 2019-12-07T19:33:58.729+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 finishing 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630000 onodes 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630000 done 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.729+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630000 [] 2019-12-07T19:33:58.729+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.729+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.729+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=20, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e765e600 con 0x5557e79cbc00 2019-12-07T19:33:58.729+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 33 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:104, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9c7a800 con 0x5557e7519800 2019-12-07T19:33:58.730+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c49a0 prio 127 cost 0 latency 0.000031 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:104, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.730+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:104, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:104, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:104, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c49a0 prio 127 cost 0 latency 0.000067 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:104, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:104, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:104 2019-12-07T19:33:58.715053+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:104 2019-12-07T19:33:58.715053+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.730+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9b11800 seq 15 2019-12-07T19:33:58.730+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 3 keys 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 3 keys = 0 2019-12-07T19:33:58.730+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.730+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11800 onodes 0x5557ea010780 shared_blobs 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore.extentmap(0x5557ea010888) update 1#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:a7686634:::smithi13140796-35:head# is 833 (485 bytes onode + 2 bytes spanning blobs + 346 bytes inline extents) 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 prepare 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11800 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 io_done 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.730+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e75c49a0 finish 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1873 < min_flush_size 524288 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea010780 had 1 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1896 < min_flush_size 524288 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1896 < min_flush_size 524288 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x482d5~768 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x482d5 0x482d5~768 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x182d5 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x2d5 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa3d and padding block with 0x5c3 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x48a3d 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.730+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000150046s (0.000000075s flush + 0.000149971s kv commit) 2019-12-07T19:33:58.730+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.730+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.730+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11800> 2019-12-07T19:33:58.730+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.730+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 kv_submitted 2019-12-07T19:33:58.730+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11800 2019-12-07T19:33:58.730+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 finishing 2019-12-07T19:33:58.730+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11800 onodes 0x5557ea010780 2019-12-07T19:33:58.730+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11800 done 2019-12-07T19:33:58.730+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.730+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11800 [] 2019-12-07T19:33:58.730+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.730+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.730+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=17, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e9ef2400 con 0x5557e7519800 2019-12-07T19:33:58.730+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 90 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d83800 con 0x5557e79cac00 2019-12-07T19:33:58.731+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5760 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.731+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.731+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 91 ==== MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9bc0a00 con 0x5557e79cac00 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.731+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.731+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c5760 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'8], crt=21'7) [] 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] rollforward: entry=21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.731+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7692c00 seq 19 2019-12-07T19:33:58.731+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692c00 onodes shared_blobs 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 prepare 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692c00 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 io_done 2019-12-07T19:33:58.731+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c5760 finish 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000167 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.731+0000 7f2b76cdf700 25 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.731+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 0x0~1000 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x22000 (139264) 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9be2a80 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5acb0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9be2a80 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5acb0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x48a3d~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x48a3d 0x48a3d~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x18a3d 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa3d 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xeae and padding block with 0x152 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x48eae 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.731+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000093015s (0.000000069s flush + 0.000092946s kv commit) 2019-12-07T19:33:58.731+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.731+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.731+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692c00> 2019-12-07T19:33:58.731+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.731+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 kv_submitted 2019-12-07T19:33:58.731+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692c00 2019-12-07T19:33:58.731+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 finishing 2019-12-07T19:33:58.731+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692c00 onodes 2019-12-07T19:33:58.731+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692c00 done 2019-12-07T19:33:58.731+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.731+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.731+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692c00 [] 2019-12-07T19:33:58.731+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.731+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9be2a80 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5acb0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 bluestore.BufferSpace(0x5557e9e5acc8 in 0x5557e6269880) _discard 0x0~1000 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 _add level 1 near 0 on buffer(0x5557e9f006c0 space 0x5557e9e5acc8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 0x0~1000 = 4096 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:e7f24557:::smithi13140796-34:head 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:e7f24557:::smithi13140796-34:head 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 139264 2019-12-07T19:33:58.731+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# hinfo_key 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# hinfo_key = 0 2019-12-07T19:33:58.731+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=62, attrs_read=0)) v2 -- 0x5557e9d59b80 con 0x5557e79cac00 2019-12-07T19:33:58.731+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e75c5b80 finish 2019-12-07T19:33:58.731+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=61, last_complete=21'8, committed=1, applied=1)) v2 -- 0x5557e9ef3a00 con 0x5557e79cac00 2019-12-07T19:33:58.731+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.732+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 92 ==== MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 157620+0+0 (crc 0 0 0) 0x5557e9c7b000 con 0x5557e79cac00 2019-12-07T19:33:58.732+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695340 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.732+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e766a000 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7695340 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:112 2019-12-07T19:33:58.727277+0000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:112 2019-12-07T19:33:58.727277+0000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.732+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e7692000 seq 9 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.es5_head 0x5557e75e34a0) get_onode oid 5#1:76d0c63f:::smithi13140796-38:head# key 0x85800000000000000176d0c6'?!smithi13140796-38!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.732+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1117 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# = 0 2019-12-07T19:33:58.732+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# 2 keys 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# 2 keys = 0 2019-12-07T19:33:58.732+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.732+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~26000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692000 19 blobs 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x26000/1000,26000,0 2019-12-07T19:33:58.732+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x27a000~26000/1000,26000,0 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x27a000~26000] 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b0a0 blob([0x27a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b1f0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9e5b0a0) get_ref 0x0~2000 Blob(0x5557e9e5b0a0 blob([0x27a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b1f0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9e5b0a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9e5b0a0 blob([0x27a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b1f0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9e5b208 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5ae70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5ae70 blob([0x27c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8cb0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9e5ae70) get_ref 0x0~2000 Blob(0x5557e9e5ae70 blob([0x27c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8cb0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9e5ae70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9e5ae70 blob([0x27c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74d8cb0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e74d8cc8 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb7ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb7ce0 blob([0x27e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7d50 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7ce0) get_ref 0x0~2000 Blob(0x5557e9eb7ce0 blob([0x27e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7d50 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9eb7ce0 blob([0x27e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb7d50 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb7d68 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb7e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb7e30 blob([0x280000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7ea0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7e30) get_ref 0x0~2000 Blob(0x5557e9eb7e30 blob([0x280000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7ea0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9eb7e30 blob([0x280000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb7ea0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb7eb8 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb7f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb7f10 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7f80 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7f10) get_ref 0x0~2000 Blob(0x5557e9eb7f10 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb7f80 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9eb7f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9eb7f10 blob([0x282000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb7f80 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9eb7f98 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f28000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f28000 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28070 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28000) get_ref 0x0~2000 Blob(0x5557e9f28000 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28070 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9f28000 blob([0x284000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28070 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28088 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f280e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f280e0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28150 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f280e0) get_ref 0x0~2000 Blob(0x5557e9f280e0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28150 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f280e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9f280e0 blob([0x286000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28150 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28168 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f281c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f281c0 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28230 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f281c0) get_ref 0x0~2000 Blob(0x5557e9f281c0 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28230 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f281c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9f281c0 blob([0x288000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28230 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28248 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f282a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f282a0 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28310 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f282a0) get_ref 0x0~2000 Blob(0x5557e9f282a0 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28310 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f282a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9f282a0 blob([0x28a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28310 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28328 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f28380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f283f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f28380 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f283f0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28380) get_ref 0x0~2000 Blob(0x5557e9f28380 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f283f0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9f28380 blob([0x28c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f283f0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28408 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f28460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f284d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f28460 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f284d0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28460) get_ref 0x0~2000 Blob(0x5557e9f28460 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f284d0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9f28460 blob([0x28e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f284d0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f284e8 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f28540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f285b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f28540 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f285b0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28540) get_ref 0x0~2000 Blob(0x5557e9f28540 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f285b0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9f28540 blob([0x290000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f285b0 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f285c8 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f28620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f28620 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28690 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28620) get_ref 0x0~2000 Blob(0x5557e9f28620 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28690 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9f28620 blob([0x292000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28690 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f286a8 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f28700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f28700 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28770 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28700) get_ref 0x0~2000 Blob(0x5557e9f28700 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28770 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9f28700 blob([0x294000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28770 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28788 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f287e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f287e0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28850 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f287e0) get_ref 0x0~2000 Blob(0x5557e9f287e0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28850 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f287e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9f287e0 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28850 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28868 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f288c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f288c0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28930 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f288c0) get_ref 0x0~2000 Blob(0x5557e9f288c0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28930 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f288c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9f288c0 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28930 sbid 0x0)) 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28948 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f289a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.732+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f289a0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28a10 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f289a0) get_ref 0x0~2000 Blob(0x5557e9f289a0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28a10 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f289a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9f289a0 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28a10 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28a28 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f28a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f28a80 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28af0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28a80) get_ref 0x0~2000 Blob(0x5557e9f28a80 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28af0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9f28a80 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28af0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28b08 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f28b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f28b60 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28bd0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28b60) get_ref 0x0~2000 Blob(0x5557e9f28b60 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f28bd0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f28b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9f28b60 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f28bd0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f28be8 in 0x5557e6269ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x26000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb4d88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 = 0 2019-12-07T19:33:58.733+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.733+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692000 cost 237270 (20 ios * 4000 + 157270 bytes) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692000 onodes 0x5557e9eb4c80 shared_blobs 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb4d88) update 5#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb4d88) update inline shard 364 bytes from 19 extents 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:76d0c63f:::smithi13140796-38:head# is 783 (413 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692000 allocated 0x[27a000~26000] released 0x[] 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 freelist allocate 0x27a000~26000 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 freelist _xor first_key 0x200000 last_key 0x280000 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 prepare 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692000 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.733+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e7695340 finish 2019-12-07T19:33:58.733+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.733+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 aio_wait 2019-12-07T19:33:58.733+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692000 2019-12-07T19:33:58.733+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 io_done 2019-12-07T19:33:58.733+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.733+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.733+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.733+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1524 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 60 ==== MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c78800 con 0x5557e79ca800 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4c80 had 1 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:58.734+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4b00 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1547 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1547 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x48eae~60b to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x48eae 0x48eae~60b to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x18eae 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xeae 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x4b9 and padding block with 0xb47 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGOpItem(op=MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e764f000 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4b00 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs5 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x494b9 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000151990s (0.000034829s flush + 0.000117161s kv commit) 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:98 2019-12-07T19:33:58.707684+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.734+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692000> 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.734+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 kv_submitted 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692000 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7692900 seq 16 2019-12-07T19:33:58.734+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 finishing 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692000 onodes 0x5557e9eb4c80 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d8cc8 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e74d8cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b208 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9cd62d0 space 0x5557e9e5b208 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb7d68 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9eb7d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb7eb8 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e9eb7eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb7f98 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9eb7f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28088 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9f28088 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28168 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9b59a70 space 0x5557e9f28168 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28248 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9b599e0 space 0x5557e9f28248 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=58, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e7a9b600 con 0x5557e79cac00 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28328 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9f28328 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692900 onodes shared_blobs 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28408 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9e44480 space 0x5557e9f28408 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f284e8 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9e44510 space 0x5557e9f284e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f285c8 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9e445a0 space 0x5557e9f285c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f286a8 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9e44630 space 0x5557e9f286a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28788 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9e44090 space 0x5557e9f28788 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28868 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9e45710 space 0x5557e9f28868 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 prepare 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28948 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9cd6360 space 0x5557e9f28948 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692900 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 io_done 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28a28 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9f28a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28b08 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9cd7b00 space 0x5557e9f28b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f28be8 in 0x5557e6269ea0) _finish_write discard buffer(0x5557e9cd7a70 space 0x5557e9f28be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.734+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c4b00 finish 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.734+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692000 done 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:58.734+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692000 [] 2019-12-07T19:33:58.734+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x494b9~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x494b9 0x494b9~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x194b9 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x4b9 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x92a and padding block with 0x6d6 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4992a 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.734+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000119375s (0.000000105s flush + 0.000119270s kv commit) 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.734+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692900> 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.734+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 kv_submitted 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692900 2019-12-07T19:33:58.734+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 finishing 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692900 onodes 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 done 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:33:58.734+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692900 [] 2019-12-07T19:33:58.734+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.734+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.734+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.734+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=33, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e9b36000 con 0x5557e79ca800 2019-12-07T19:33:58.734+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.736+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 34 ==== MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=19, reqid=client.4540.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e70a7000 con 0x5557e7519800 2019-12-07T19:33:58.736+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e75c42c0 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=19, reqid=client.4540.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.736+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=19, reqid=client.4540.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=19, reqid=client.4540.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.736+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 35 ==== MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=20, to_read={1:a7686634:::smithi13140796-35:head=0,4096,0}, subchunks={1:a7686634:::smithi13140796-35:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9b94000 con 0x5557e7519800 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=19, reqid=client.4540.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.736+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695760 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=20, to_read={1:a7686634:::smithi13140796-35:head=0,4096,0}, subchunks={1:a7686634:::smithi13140796-35:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.736+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=20, to_read={1:a7686634:::smithi13140796-35:head=0,4096,0}, subchunks={1:a7686634:::smithi13140796-35:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 osd.6 21 dequeue_op 0x5557e75c42c0 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=19, reqid=client.4540.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.5s1 21/18 ECSubWrite(tid=19, reqid=client.4540.0:104, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=20, to_read={1:a7686634:::smithi13140796-35:head=0,4096,0}, subchunks={1:a7686634:::smithi13140796-35:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:104 2019-12-07T19:33:58.715053+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.736+0000 7f2b77ce1700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7665200 seq 16 2019-12-07T19:33:58.736+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7665200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7665200 onodes shared_blobs 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7665200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 prepare 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7665200 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 io_done 2019-12-07T19:33:58.736+0000 7f2b77ce1700 10 osd.6 21 dequeue_op 0x5557e75c42c0 finish 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=20, to_read={1:a7686634:::smithi13140796-35:head=0,4096,0}, subchunks={1:a7686634:::smithi13140796-35:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e764c000 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e7695760 prio 127 cost 0 latency 0.000125 MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=20, to_read={1:a7686634:::smithi13140796-35:head=0,4096,0}, subchunks={1:a7686634:::smithi13140796-35:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.5s1 21/18 ECSubRead(tid=20, to_read={1:a7686634:::smithi13140796-35:head=0,4096,0}, subchunks={1:a7686634:::smithi13140796-35:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4992a~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.736+0000 7f2b73cd9700 25 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4992a 0x4992a~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.736+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 0x0~1000 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x23000 (143360) 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1992a 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x92a 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xd9b and padding block with 0x265 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9e5abd0 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a930 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9e5abd0 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a930 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x49d9b 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.736+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000132390s (0.000000078s flush + 0.000132312s kv commit) 2019-12-07T19:33:58.736+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.736+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7665200> 2019-12-07T19:33:58.736+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.736+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.736+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 kv_submitted 2019-12-07T19:33:58.736+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7665200 2019-12-07T19:33:58.736+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7665200 finishing 2019-12-07T19:33:58.736+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7665200 onodes 2019-12-07T19:33:58.736+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7665200 done 2019-12-07T19:33:58.736+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:33:58.736+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7665200 [] 2019-12-07T19:33:58.736+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.736+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9e5abd0 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a930 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b7a948 in 0x5557e6269180) _discard 0x0~1000 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 _add level 1 near 0 on buffer(0x5557e9e44000 space 0x5557e9b7a948 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 0x0~1000 = 4096 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:a7686634:::smithi13140796-35:head 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:a7686634:::smithi13140796-35:head 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 143360 2019-12-07T19:33:58.736+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# hinfo_key 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# hinfo_key = 0 2019-12-07T19:33:58.736+0000 7f2b73cd9700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpReadReply(1.5s0 21/18 ECSubReadReply(tid=20, attrs_read=0)) v2 -- 0x5557e9b7d180 con 0x5557e7519800 2019-12-07T19:33:58.736+0000 7f2b73cd9700 10 osd.6 21 dequeue_op 0x5557e7695760 finish 2019-12-07T19:33:58.736+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.736+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 21/18 ECSubWriteReply(tid=19, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e9ef2600 con 0x5557e7519800 2019-12-07T19:33:58.736+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.739+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 61 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 169908+0+0 (crc 0 0 0) 0x5557e9c7a800 con 0x5557e79ca800 2019-12-07T19:33:58.739+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.739+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:118, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'8], crt=21'8) [21'9 (0'0) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:118 2019-12-07T19:33:58.736313+0000 0 ObjectCleanRegions clean_offsets: [671744~18446744073708879871], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] add_log_entry 21'9 (0'0) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:118 2019-12-07T19:33:58.736313+0000 0 ObjectCleanRegions clean_offsets: [671744~18446744073708879871], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.739+0000 7f2b774e0700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7121500 seq 16 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) get_onode oid 1#1:639abb38:::smithi13140796-40:head# key 0x818000000000000001639abb'8!smithi13140796-40!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.739+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1118 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# = 0 2019-12-07T19:33:58.739+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 2 keys 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 2 keys = 0 2019-12-07T19:33:58.739+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.739+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 0x0~29000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:639abb38:::smithi13140796-40:head# 0x0~29000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~29000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7121500 21 blobs 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 fbmap_alloc 0x5557e6282e00 allocate 0x29000/1000,29000,0 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2a0000~29000/1000,29000,0 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2a0000~29000] 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb3340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0bea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb3340 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0bea0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9cb3340) get_ref 0x0~2000 Blob(0x5557e9cb3340 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0bea0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9cb3340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9cb3340 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0bea0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9f0beb8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb3dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb3dc0 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3500 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9cb3dc0) get_ref 0x0~2000 Blob(0x5557e9cb3dc0 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3500 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9cb3dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9cb3dc0 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3500 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9cb3518 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d372d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d372d0 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37340 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d372d0) get_ref 0x0~2000 Blob(0x5557e9d372d0 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37340 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d372d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9d372d0 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37340 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9d37358 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d373b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d373b0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37420 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d373b0) get_ref 0x0~2000 Blob(0x5557e9d373b0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37420 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9d373b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9d373b0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37420 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9d37438 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0000 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0070 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0000) get_ref 0x0~2000 Blob(0x5557e9ec0000 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0070 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9ec0000 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0070 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0088 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec00e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec00e0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0150 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec00e0) get_ref 0x0~2000 Blob(0x5557e9ec00e0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0150 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec00e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9ec00e0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0150 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0168 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec01c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec01c0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0230 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec01c0) get_ref 0x0~2000 Blob(0x5557e9ec01c0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0230 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec01c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9ec01c0 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0230 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0248 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec02a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec02a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0310 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec02a0) get_ref 0x0~2000 Blob(0x5557e9ec02a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0310 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec02a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9ec02a0 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0310 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0328 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec03f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0380 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec03f0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0380) get_ref 0x0~2000 Blob(0x5557e9ec0380 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec03f0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9ec0380 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec03f0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0408 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec04d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0460 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec04d0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0460) get_ref 0x0~2000 Blob(0x5557e9ec0460 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec04d0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9ec0460 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec04d0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec04e8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec05b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0540 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec05b0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0540) get_ref 0x0~2000 Blob(0x5557e9ec0540 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec05b0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9ec0540 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec05b0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec05c8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0620 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0690 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0620) get_ref 0x0~2000 Blob(0x5557e9ec0620 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0690 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9ec0620 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0690 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec06a8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0700 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0770 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0700) get_ref 0x0~2000 Blob(0x5557e9ec0700 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0770 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9ec0700 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0770 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0788 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec07e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec07e0 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0850 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec07e0) get_ref 0x0~2000 Blob(0x5557e9ec07e0 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0850 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec07e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9ec07e0 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0850 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0868 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec08c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec08c0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0930 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec08c0) get_ref 0x0~2000 Blob(0x5557e9ec08c0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0930 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec08c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9ec08c0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0930 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0948 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec09a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec09a0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0a10 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec09a0) get_ref 0x0~2000 Blob(0x5557e9ec09a0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0a10 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec09a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9ec09a0 blob([0x2be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0a10 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0a28 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0a80 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0af0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0a80) get_ref 0x0~2000 Blob(0x5557e9ec0a80 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0af0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9ec0a80 blob([0x2c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0af0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0b08 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0b60 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0bd0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0b60) get_ref 0x0~2000 Blob(0x5557e9ec0b60 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0bd0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9ec0b60 blob([0x2c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0bd0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0be8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0c40 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0cb0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0c40) get_ref 0x0~2000 Blob(0x5557e9ec0c40 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0cb0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9ec0c40 blob([0x2c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0cb0 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0cc8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0d20 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0d90 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0d20) get_ref 0x0~2000 Blob(0x5557e9ec0d20 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0d90 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e9ec0d20 blob([0x2c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec0d90 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0da8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec0e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec0e00 blob([0x2c8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0e70 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0e00) get_ref 0x0~1000 Blob(0x5557e9ec0e00 blob([0x2c8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec0e70 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.blob(0x5557e9ec0e00) get_ref init 0x1000, 1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~1000: 0x0~1000 Blob(0x5557e9ec0e00 blob([0x2c8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9ec0e70 sbid 0x0)) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9ec0e88 in 0x5557e6268fc0) _discard 0x0~1000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x29000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e9d58888) dirty_range mark inline shard dirty 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 0x0~29000 = 0 2019-12-07T19:33:58.739+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.739+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121500 cost 257558 (22 ios * 4000 + 169558 bytes) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121500 onodes 0x5557e9d58780 shared_blobs 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e9d58888) update 1#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e9d58888) update inline shard 399 bytes from 21 extents 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:639abb38:::smithi13140796-40:head# is 818 (413 bytes onode + 2 bytes spanning blobs + 403 bytes inline extents) 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121500 allocated 0x[2a0000~29000] released 0x[] 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 freelist allocate 0x2a0000~29000 2019-12-07T19:33:58.739+0000 7f2b774e0700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 prepare 2019-12-07T19:33:58.739+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7121500 2019-12-07T19:33:58.740+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.740+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c5b80 finish 2019-12-07T19:33:58.740+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.740+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 aio_wait 2019-12-07T19:33:58.740+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121500 2019-12-07T19:33:58.740+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 io_done 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1530 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9d58780 had 1 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1553 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1553 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x49d9b~611 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x49d9b 0x49d9b~611 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x19d9b 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xd9b 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x3ac and padding block with 0xc54 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4a3ac 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.740+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000157382s (0.000048487s flush + 0.000108895s kv commit) 2019-12-07T19:33:58.740+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121500> 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.740+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121500 2019-12-07T19:33:58.740+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 finishing 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121500 onodes 0x5557e9d58780 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb3518 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b58630 space 0x5557e9cb3518 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37358 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9d37358 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37438 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9d37438 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0088 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b59a70 space 0x5557e9ec0088 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0168 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b599e0 space 0x5557e9ec0168 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0248 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9ec0248 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0328 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9ec0328 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0408 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9ec0408 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec04e8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9ec04e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec05c8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7c20 space 0x5557e9ec05c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec06a8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7cb0 space 0x5557e9ec06a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0788 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9ec0788 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0868 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e9ec0868 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0948 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd62d0 space 0x5557e9ec0948 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0a28 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9ec0a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0b08 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e9ec0b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0be8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6360 space 0x5557e9ec0be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0cc8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9ec0cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0da8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7b00 space 0x5557e9ec0da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec0e88 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd7a70 space 0x5557e9ec0e88 0x0~1000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0beb8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9b59950 space 0x5557e9f0beb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 done 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.740+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121500 [] 2019-12-07T19:33:58.740+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.740+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.740+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=35, last_complete=21'9, committed=1, applied=1)) v2 -- 0x5557e765e600 con 0x5557e79ca800 2019-12-07T19:33:58.740+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.741+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 62 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c7a000 con 0x5557e79ca800 2019-12-07T19:33:58.741+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.741+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000040 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=36, reqid=client.4540.0:118, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'9], crt=21'8) [] 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] rollforward: entry=21'9 (0'0) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:118 2019-12-07T19:33:58.736313+0000 0 ObjectCleanRegions clean_offsets: [671744~18446744073708879871], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.741+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7692f00 seq 17 2019-12-07T19:33:58.741+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692f00 onodes shared_blobs 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 prepare 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692f00 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 io_done 2019-12-07T19:33:58.741+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4000 finish 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4a3ac~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4a3ac 0x4a3ac~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1a3ac 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x3ac 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x81d and padding block with 0x7e3 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4a81d 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.741+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000110293s (0.000000063s flush + 0.000110230s kv commit) 2019-12-07T19:33:58.741+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.741+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.741+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692f00> 2019-12-07T19:33:58.741+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.741+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 kv_submitted 2019-12-07T19:33:58.741+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692f00 2019-12-07T19:33:58.741+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 finishing 2019-12-07T19:33:58.741+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692f00 onodes 2019-12-07T19:33:58.741+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692f00 done 2019-12-07T19:33:58.741+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.741+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692f00 [] 2019-12-07T19:33:58.741+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.741+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.741+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.741+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=36, last_complete=21'9, committed=1, applied=1)) v2 -- 0x5557e9b36000 con 0x5557e79ca800 2019-12-07T19:33:58.741+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.741+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.2 172.21.15.131:0/37077 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.214487565s delta_ub 0.017613708s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e765e600 con 0x5557e74ccc00 2019-12-07T19:33:58.741+0000 7f2b957b2700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.017148141s,0.017613708s]) 2019-12-07T19:33:58.741+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.197339424s delta_ub 18446744073.692403475s) v5 -- 0x5557e76b1e00 con 0x5557e74ccc00 2019-12-07T19:33:58.741+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.2 172.21.15.131:0/37077 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.214487565s delta_ub 0.017613708s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e758c200 con 0x5557e74cd400 2019-12-07T19:33:58.741+0000 7f2b967b4700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016907369s,0.017613708s]) 2019-12-07T19:33:58.741+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.197580196s delta_ub 18446744073.692644247s) v5 -- 0x5557e9d89a00 con 0x5557e74cd400 2019-12-07T19:33:58.742+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 63 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9c7b800 con 0x5557e79ca800 2019-12-07T19:33:58.742+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c54a0 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.742+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c54a0 prio 127 cost 0 latency 0.000046 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=37, reqid=client.4540.0:119, at_version=21'10, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log log((0'0,21'9], crt=21'9) [21'10 (21'9) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:119 2019-12-07T19:33:58.736345+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] add_log_entry 21'10 (21'9) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:119 2019-12-07T19:33:58.736345+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] append_log approx pg log length = 10 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.742+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e9cc4900 seq 18 2019-12-07T19:33:58.742+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 3 keys 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 3 keys = 0 2019-12-07T19:33:58.742+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.742+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4900 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4900 onodes 0x5557e9d58780 shared_blobs 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9d58888) update 1#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:639abb38:::smithi13140796-40:head# is 890 (485 bytes onode + 2 bytes spanning blobs + 403 bytes inline extents) 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 prepare 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4900 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 io_done 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] op_applied version 21'10 2019-12-07T19:33:58.742+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c54a0 finish 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1930 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9d58780 had 1 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1953 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1953 < min_flush_size 524288 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4a81d~7a1 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4a81d 0x4a81d~7a1 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1a81d 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x81d 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xfbe and padding block with 0x42 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4afbe 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.742+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000116555s (0.000000064s flush + 0.000116491s kv commit) 2019-12-07T19:33:58.742+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.742+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.742+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4900> 2019-12-07T19:33:58.742+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.742+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 kv_submitted 2019-12-07T19:33:58.742+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4900 2019-12-07T19:33:58.742+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4900 finishing 2019-12-07T19:33:58.742+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4900 onodes 0x5557e9d58780 2019-12-07T19:33:58.742+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4900 done 2019-12-07T19:33:58.742+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.742+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4900 [] 2019-12-07T19:33:58.742+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.742+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.742+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=37, last_complete=21'10, committed=1, applied=1)) v2 -- 0x5557e9d88200 con 0x5557e79ca800 2019-12-07T19:33:58.742+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.743+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 93 ==== MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9ba0000 con 0x5557e79cac00 2019-12-07T19:33:58.743+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4840 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.743+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e766a000 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4840 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:112 2019-12-07T19:33:58.727277+0000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.743+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e9cc4f00 seq 10 2019-12-07T19:33:58.743+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4f00 onodes shared_blobs 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4f00 prepare 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4f00 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4f00 io_done 2019-12-07T19:33:58.743+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4840 finish 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4afbe~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4afbe 0x4afbe~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1afbe 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xfbe 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x42f and padding block with 0xbd1 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4b42f 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000129971s (0.000000062s flush + 0.000129909s kv commit) 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.743+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 94 ==== MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9b9e000 con 0x5557e79cac00 2019-12-07T19:33:58.743+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c49a0 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.743+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e766a000 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c49a0 prio 127 cost 0 latency 0.000044 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:113 2019-12-07T19:33:58.727310+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4f00> 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.743+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4f00 kv_submitted 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4f00 2019-12-07T19:33:58.743+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4f00 finishing 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4f00 onodes 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4f00 done 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:58.743+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4f00 [] 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:113 2019-12-07T19:33:58.727310+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.743+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.743+0000 7f2b774e0700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e7693800 seq 11 2019-12-07T19:33:58.743+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# 3 keys 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# 3 keys = 0 2019-12-07T19:33:58.743+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.es5_head 5#1:76d0c63f:::smithi13140796-38:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.743+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693800 onodes 0x5557e9eb4c80 shared_blobs 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore.extentmap(0x5557e9eb4d88) update 5#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:76d0c63f:::smithi13140796-38:head# is 855 (485 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 prepare 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693800 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 io_done 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.743+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c49a0 finish 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1895 < min_flush_size 524288 2019-12-07T19:33:58.743+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=65, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e9b36000 con 0x5557e79cac00 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4c80 had 1 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1918 < min_flush_size 524288 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1918 < min_flush_size 524288 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4b42f~77e to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4b42f 0x4b42f~77e to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1b42f 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x42f 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xbad and padding block with 0x453 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4bbad 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.743+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000120587s (0.000000063s flush + 0.000120524s kv commit) 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.743+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693800> 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.743+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 kv_submitted 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693800 2019-12-07T19:33:58.743+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 finishing 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693800 onodes 0x5557e9eb4c80 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693800 done 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:58.743+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693800 [] 2019-12-07T19:33:58.743+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.743+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.743+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.743+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=66, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e9d88000 con 0x5557e79cac00 2019-12-07T19:33:58.743+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.744+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 247176 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.744+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 95 ==== MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9b9f000 con 0x5557e79cac00 2019-12-07T19:33:58.744+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4dc0 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.744+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGOpItem(op=MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e766a000 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4dc0 prio 127 cost 0 latency 0.000041 MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es5 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:113 2019-12-07T19:33:58.727310+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.744+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557e7046f00 seq 12 2019-12-07T19:33:58.744+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7046f00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7046f00 onodes shared_blobs 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7046f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 prepare 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7046f00 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 io_done 2019-12-07T19:33:58.744+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4dc0 finish 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4bbad~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4bbad 0x4bbad~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1bbad 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xbad 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x1e and padding block with 0xfe2 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4c01e 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.744+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000099652s (0.000000064s flush + 0.000099588s kv commit) 2019-12-07T19:33:58.744+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.744+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.744+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7046f00> 2019-12-07T19:33:58.744+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.744+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 kv_submitted 2019-12-07T19:33:58.744+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7046f00 2019-12-07T19:33:58.744+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7046f00 finishing 2019-12-07T19:33:58.744+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7046f00 onodes 2019-12-07T19:33:58.744+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7046f00 done 2019-12-07T19:33:58.744+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:33:58.744+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7046f00 [] 2019-12-07T19:33:58.744+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.744+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.744+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.744+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=67, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e765e600 con 0x5557e79cac00 2019-12-07T19:33:58.744+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 64 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9fca000 con 0x5557e79ca800 2019-12-07T19:33:58.745+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5080 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.745+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.745+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 65 ==== MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9d59b80 con 0x5557e79ca800 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.745+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.745+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c5080 prio 127 cost 0 latency 0.000053 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2 pg pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=38, reqid=client.4540.0:119, at_version=0'0, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log log((0'0,21'10], crt=21'9) [] 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] rollforward: entry=21'10 (21'9) modify 1:639abb38:::smithi13140796-40:head by client.4540.0:119 2019-12-07T19:33:58.736345+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log approx pg log length = 10 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.745+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7a92600 seq 19 2019-12-07T19:33:58.745+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92600 onodes shared_blobs 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 prepare 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92600 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 io_done 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.745+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c5080 finish 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4c01e~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4c01e 0x4c01e~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1c01e 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000132 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x1e 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x48f and padding block with 0xb71 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=39, to_read={1:639abb38:::smithi13140796-40:head=0,4096,0}, subchunks={1:639abb38:::smithi13140796-40:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.745+0000 7f2b774e0700 25 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.745+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 0x0~1000 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x29000 (167936) 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9cb3340 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0bea0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9cb3340 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0bea0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4c48f 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.745+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000109049s (0.000000062s flush + 0.000108987s kv commit) 2019-12-07T19:33:58.745+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.745+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.745+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92600> 2019-12-07T19:33:58.745+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.745+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 kv_submitted 2019-12-07T19:33:58.745+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92600 2019-12-07T19:33:58.745+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 finishing 2019-12-07T19:33:58.745+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92600 onodes 2019-12-07T19:33:58.745+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92600 done 2019-12-07T19:33:58.745+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.745+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92600 [] 2019-12-07T19:33:58.745+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.745+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9cb3340 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0bea0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 bluestore.BufferSpace(0x5557e9f0beb8 in 0x5557e6268fc0) _discard 0x0~1000 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 _add level 1 near 0 on buffer(0x5557e9e457a0 space 0x5557e9f0beb8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 0x0~1000 = 4096 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] get_hash_info: Getting attr on 1:639abb38:::smithi13140796-40:head 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] get_hash_info: not in cache 1:639abb38:::smithi13140796-40:head 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] get_hash_info: found on disk, size 167936 2019-12-07T19:33:58.745+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# hinfo_key 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:639abb38:::smithi13140796-40:head# hinfo_key = 0 2019-12-07T19:33:58.745+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=39, attrs_read=0)) v2 -- 0x5557e9d58000 con 0x5557e79ca800 2019-12-07T19:33:58.745+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e75c5e40 finish 2019-12-07T19:33:58.745+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=38, last_complete=21'10, committed=1, applied=1)) v2 -- 0x5557e9d89200 con 0x5557e79ca800 2019-12-07T19:33:58.745+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.747+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 96 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=69, reqid=client.4540.0:121, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 174004+0+0 (crc 0 0 0) 0x5557e9fcb000 con 0x5557e79cac00 2019-12-07T19:33:58.747+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4580 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=69, reqid=client.4540.0:121, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.747+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=69, reqid=client.4540.0:121, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.747+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.747+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=69, reqid=client.4540.0:121, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.747+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.747+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=69, reqid=client.4540.0:121, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.747+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4580 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=69, reqid=client.4540.0:121, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.747+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=69, reqid=client.4540.0:121, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.747+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'8 (0'0,21'8] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'8], crt=21'8) [21'9 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:121 2019-12-07T19:33:58.741112+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.747+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] add_log_entry 21'9 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:121 2019-12-07T19:33:58.741112+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.747+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.748+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.748+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.748+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9b10300 seq 19 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:14fc029d:::smithi13140796-41:head# key 0x85800000000000000114fc029d21736d'ithi13140796-41!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.748+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1119 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# = 0 2019-12-07T19:33:58.748+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 2 keys 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 2 keys = 0 2019-12-07T19:33:58.748+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.748+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 0x0~2a000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:14fc029d:::smithi13140796-41:head# 0x0~2a000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~2a000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10300 21 blobs 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.748+0000 7f2b764de700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2a000/1000,2a000,0 2019-12-07T19:33:58.748+0000 7f2b764de700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2c9000~2a000/1000,2a000,0 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2c9000~2a000] 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2b420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec19d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2b420 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec19d0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b420) get_ref 0x0~2000 Blob(0x5557e9c2b420 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec19d0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9c2b420 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec19d0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec19e8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2b490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2b490 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6bd0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b490) get_ref 0x0~2000 Blob(0x5557e9c2b490 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c6bd0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9c2b490 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c6bd0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e75c6be8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2b570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2b570 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b5e0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b570) get_ref 0x0~2000 Blob(0x5557e9c2b570 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b5e0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9c2b570 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b5e0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b5f8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2b650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2b650 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b6c0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b650) get_ref 0x0~2000 Blob(0x5557e9c2b650 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b6c0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9c2b650 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b6c0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b6d8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2b730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2b730 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b7a0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b730) get_ref 0x0~2000 Blob(0x5557e9c2b730 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2b7a0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9c2b730 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b7a0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b7b8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1a40 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1a40) get_ref 0x0~2000 Blob(0x5557e9ec1a40 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9ec1a40 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1ac8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1b20 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1b90 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1b20) get_ref 0x0~2000 Blob(0x5557e9ec1b20 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1b90 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9ec1b20 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1b90 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1ba8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1c00 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1c70 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1c00) get_ref 0x0~2000 Blob(0x5557e9ec1c00 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1c70 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9ec1c00 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1c70 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1c88 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1ce0 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1d50 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1ce0) get_ref 0x0~2000 Blob(0x5557e9ec1ce0 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1d50 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9ec1ce0 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1d50 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1d68 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1dc0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1e30 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1dc0) get_ref 0x0~2000 Blob(0x5557e9ec1dc0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1e30 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9ec1dc0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1e30 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1e48 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1ea0 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37260 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1ea0) get_ref 0x0~2000 Blob(0x5557e9ec1ea0 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d37260 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9ec1ea0 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37260 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9d37278 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b110 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1f10 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9e5b110) get_ref 0x0~2000 Blob(0x5557e9e5b110 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1f10 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9e5b110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9e5b110 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1f10 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1f28 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1f80 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08000 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1f80) get_ref 0x0~2000 Blob(0x5557e9ec1f80 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08000 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9ec1f80 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08000 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08018 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f08070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f080e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f08070 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f080e0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08070) get_ref 0x0~2000 Blob(0x5557e9f08070 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f080e0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9f08070 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f080e0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f080f8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f08150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f081c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f08150 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f081c0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08150) get_ref 0x0~2000 Blob(0x5557e9f08150 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f081c0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9f08150 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f081c0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f081d8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f08230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f082a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f08230 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f082a0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08230) get_ref 0x0~2000 Blob(0x5557e9f08230 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f082a0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9f08230 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f082a0 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f082b8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f08310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f08310 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08380 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08310) get_ref 0x0~2000 Blob(0x5557e9f08310 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08380 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9f08310 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08380 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08398 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f083f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f083f0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08460 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f083f0) get_ref 0x0~2000 Blob(0x5557e9f083f0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08460 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f083f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9f083f0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08460 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08478 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f084d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f084d0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08540 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f084d0) get_ref 0x0~2000 Blob(0x5557e9f084d0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08540 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f084d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9f084d0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08540 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08558 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f085b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f085b0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08620 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f085b0) get_ref 0x0~2000 Blob(0x5557e9f085b0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08620 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f085b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e9f085b0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08620 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08638 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f08690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f08690 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08700 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08690) get_ref 0x0~2000 Blob(0x5557e9f08690 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08700 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x5557e9f08690 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08700 sbid 0x0)) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08718 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x2a000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 0x0~2a000 = 0 2019-12-07T19:33:58.748+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.748+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10300 cost 261654 (22 ios * 4000 + 173654 bytes) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10300 onodes 0x5557ea011180 shared_blobs 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011288) update 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011288) update inline shard 402 bytes from 21 extents 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:14fc029d:::smithi13140796-41:head# is 821 (413 bytes onode + 2 bytes spanning blobs + 406 bytes inline extents) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10300 allocated 0x[2c9000~2a000] released 0x[] 2019-12-07T19:33:58.748+0000 7f2b764de700 10 freelist allocate 0x2c9000~2a000 2019-12-07T19:33:58.748+0000 7f2b764de700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 prepare 2019-12-07T19:33:58.748+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b10300 2019-12-07T19:33:58.748+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.748+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4580 finish 2019-12-07T19:33:58.748+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.748+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 aio_wait 2019-12-07T19:33:58.748+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10300 2019-12-07T19:33:58.748+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 io_done 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1533 < min_flush_size 524288 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea011180 had 1 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1556 < min_flush_size 524288 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1556 < min_flush_size 524288 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4c48f~614 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4c48f 0x4c48f~614 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1c48f 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x48f 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xaa3 and padding block with 0x55d 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4caa3 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.748+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10300> 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000124974s (0.000042945s flush + 0.000082029s kv commit) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.748+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 kv_submitted 2019-12-07T19:33:58.748+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10300 2019-12-07T19:33:58.748+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 finishing 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10300 onodes 0x5557ea011180 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c6be8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b59ef0 space 0x5557e75c6be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2b5f8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f007e0 space 0x5557e9c2b5f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2b6d8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00120 space 0x5557e9c2b6d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2b7b8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f001b0 space 0x5557e9c2b7b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d37278 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00360 space 0x5557e9d37278 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec19e8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b59e60 space 0x5557e9ec19e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1ac8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00240 space 0x5557e9ec1ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1ba8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e445a0 space 0x5557e9ec1ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1c88 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e44510 space 0x5557e9ec1c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=69, last_complete=21'9, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cac00 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1d68 in 0x5557e6269960) _finish_write discard buffer(0x5557e9e44480 space 0x5557e9ec1d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1e48 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f002d0 space 0x5557e9ec1e48 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1f28 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f003f0 space 0x5557e9ec1f28 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f08018 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00480 space 0x5557e9f08018 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f080f8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00510 space 0x5557e9f080f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f081d8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00000 space 0x5557e9f081d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f082b8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00090 space 0x5557e9f082b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f08398 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00870 space 0x5557e9f08398 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f08478 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00900 space 0x5557e9f08478 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f08558 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00990 space 0x5557e9f08558 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f08638 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00a20 space 0x5557e9f08638 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f08718 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00ab0 space 0x5557e9f08718 0x0~2000 writing nocache) 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10300 done 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.748+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10300 [] 2019-12-07T19:33:58.748+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.748+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.751+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 97 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=70, reqid=client.4540.0:124, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 178100+0+0 (crc 0 0 0) 0x5557e9fcb800 con 0x5557e79cac00 2019-12-07T19:33:58.751+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=70, reqid=client.4540.0:124, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.751+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=70, reqid=client.4540.0:124, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.751+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=70, reqid=client.4540.0:124, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.751+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.751+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=70, reqid=client.4540.0:124, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.751+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=70, reqid=client.4540.0:124, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.8s5( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.751+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'9 (0'0,21'9] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=70, reqid=client.4540.0:124, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.751+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'9 (0'0,21'9] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'9], crt=21'8) [21'10 (0'0) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:124 2019-12-07T19:33:58.745795+0000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.751+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'9 crt=21'8 lcod 21'9 active mbc={}] add_log_entry 21'10 (0'0) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:124 2019-12-07T19:33:58.745795+0000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.751+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'9 crt=21'8 lcod 21'9 active mbc={}] append_log approx pg log length = 10 2019-12-07T19:33:58.751+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'9 crt=21'8 lcod 21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.751+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.751+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9b10c00 seq 20 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:10f769a3:::smithi13140796-42:head# key 0x85800000000000000110f769a321736d'ithi13140796-42!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.751+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1120 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# = 0 2019-12-07T19:33:58.751+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 2 keys 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 2 keys = 0 2019-12-07T19:33:58.751+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.751+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~2b000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10c00 22 blobs 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.751+0000 7f2b764de700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2b000/1000,2b000,0 2019-12-07T19:33:58.751+0000 7f2b764de700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2f3000~2b000/1000,2b000,0 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2f3000~2b000] 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b3b0 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b570 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9e5b3b0) get_ref 0x0~2000 Blob(0x5557e9e5b3b0 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b570 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9e5b3b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9e5b3b0 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b570 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9e5b588 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f090a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f090a0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09110 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f090a0) get_ref 0x0~2000 Blob(0x5557e9f090a0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09110 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f090a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9f090a0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09110 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09128 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f091f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f091f0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09260 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f091f0) get_ref 0x0~2000 Blob(0x5557e9f091f0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09260 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f091f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9f091f0 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09260 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09278 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f092d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f092d0 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09340 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f092d0) get_ref 0x0~2000 Blob(0x5557e9f092d0 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09340 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f092d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9f092d0 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09340 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09358 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f093b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f093b0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09420 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f093b0) get_ref 0x0~2000 Blob(0x5557e9f093b0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09420 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f093b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9f093b0 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09420 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09438 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09490 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09500 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09490) get_ref 0x0~2000 Blob(0x5557e9f09490 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09500 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9f09490 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09500 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09518 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f095e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09570 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f095e0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09570) get_ref 0x0~2000 Blob(0x5557e9f09570 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f095e0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9f09570 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f095e0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f095f8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f096c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09650 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f096c0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09650) get_ref 0x0~2000 Blob(0x5557e9f09650 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f096c0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9f09650 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f096c0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f096d8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f097a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09730 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f097a0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09730) get_ref 0x0~2000 Blob(0x5557e9f09730 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f097a0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9f09730 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f097a0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f097b8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09810 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09880 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09810) get_ref 0x0~2000 Blob(0x5557e9f09810 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09880 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9f09810 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09880 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09898 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f098f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f098f0 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09960 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f098f0) get_ref 0x0~2000 Blob(0x5557e9f098f0 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09960 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f098f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9f098f0 blob([0x307000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09960 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09978 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f099d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f099d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09a40 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f099d0) get_ref 0x0~2000 Blob(0x5557e9f099d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09a40 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f099d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9f099d0 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09a40 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09a58 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09ab0 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09b20 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09ab0) get_ref 0x0~2000 Blob(0x5557e9f09ab0 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09b20 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9f09ab0 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09b20 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09b38 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09b90 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09c00 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09b90) get_ref 0x0~2000 Blob(0x5557e9f09b90 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09c00 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9f09b90 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09c00 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09c18 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09ce0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09c70) get_ref 0x0~2000 Blob(0x5557e9f09c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09ce0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9f09c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09ce0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09cf8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09d50 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09dc0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09d50) get_ref 0x0~2000 Blob(0x5557e9f09d50 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09dc0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9f09d50 blob([0x311000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09dc0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09dd8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09e30 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09ea0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09e30) get_ref 0x0~2000 Blob(0x5557e9f09e30 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09ea0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9f09e30 blob([0x313000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09ea0 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09eb8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f09f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f09f10 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09f80 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09f10) get_ref 0x0~2000 Blob(0x5557e9f09f10 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f09f80 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9f09f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9f09f10 blob([0x315000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f09f80 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f09f98 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba2000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba2000 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2070 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9ba2000) get_ref 0x0~2000 Blob(0x5557e9ba2000 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2070 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9ba2000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9ba2000 blob([0x317000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba2070 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ba2088 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba20e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba20e0 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2150 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9ba20e0) get_ref 0x0~2000 Blob(0x5557e9ba20e0 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2150 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9ba20e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e9ba20e0 blob([0x319000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba2150 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ba2168 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba21c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba21c0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2230 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9ba21c0) get_ref 0x0~2000 Blob(0x5557e9ba21c0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2230 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9ba21c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x5557e9ba21c0 blob([0x31b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba2230 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ba2248 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba22a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba22a0 blob([0x31d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2310 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9ba22a0) get_ref 0x0~1000 Blob(0x5557e9ba22a0 blob([0x31d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2310 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.blob(0x5557e9ba22a0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2a000~1000: 0x0~1000 Blob(0x5557e9ba22a0 blob([0x31d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9ba2310 sbid 0x0)) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ba2328 in 0x5557e6269960) _discard 0x0~1000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x2b000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 = 0 2019-12-07T19:33:58.751+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.751+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10c00 cost 269750 (23 ios * 4000 + 177750 bytes) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10c00 onodes 0x5557ea011400 shared_blobs 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011508) update 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011508) update inline shard 418 bytes from 22 extents 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:10f769a3:::smithi13140796-42:head# is 837 (413 bytes onode + 2 bytes spanning blobs + 422 bytes inline extents) 2019-12-07T19:33:58.751+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10c00 allocated 0x[2f3000~2b000] released 0x[] 2019-12-07T19:33:58.751+0000 7f2b764de700 10 freelist allocate 0x2f3000~2b000 2019-12-07T19:33:58.751+0000 7f2b764de700 20 freelist _xor first_key 0x280000 last_key 0x300000 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 prepare 2019-12-07T19:33:58.751+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b10c00 2019-12-07T19:33:58.752+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'9 crt=21'8 lcod 21'9 active mbc={}] op_applied version 21'10 2019-12-07T19:33:58.752+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e75c4f20 finish 2019-12-07T19:33:58.752+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.752+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 6 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (crc 0 0 0) 0x5557e6289200 con 0x5557e79cb800 2019-12-07T19:33:58.752+0000 7f2b834f8700 20 osd.6 21 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.752+0000 7f2b834f8700 10 osd.6 21 do_waiters -- start 2019-12-07T19:33:58.752+0000 7f2b834f8700 10 osd.6 21 do_waiters -- finish 2019-12-07T19:33:58.752+0000 7f2b834f8700 20 osd.6 21 _dispatch 0x5557e6289200 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.752+0000 7f2b834f8700 3 osd.6 21 handle_osd_map epochs [20,21], i have 21, src has [1,21] 2019-12-07T19:33:58.752+0000 7f2b834f8700 10 osd.6 21 no new maps here, dropping 2019-12-07T19:33:58.752+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 aio_wait 2019-12-07T19:33:58.752+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10c00 2019-12-07T19:33:58.752+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 io_done 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1578 < min_flush_size 524288 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea011400 had 1 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1601 < min_flush_size 524288 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1601 < min_flush_size 524288 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4caa3~641 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4caa3 0x4caa3~641 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1caa3 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xaa3 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe4 and padding block with 0xf1c 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4d0e4 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.752+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000149203s (0.000040156s flush + 0.000109047s kv commit) 2019-12-07T19:33:58.752+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10c00> 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.752+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10c00 2019-12-07T19:33:58.752+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 finishing 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10c00 onodes 0x5557ea011400 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba2088 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b59950 space 0x5557e9ba2088 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba2168 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e9ba2168 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba2248 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd6360 space 0x5557e9ba2248 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba2328 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9ba2328 0x0~1000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b588 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00c60 space 0x5557e9e5b588 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09128 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00cf0 space 0x5557e9f09128 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09278 in 0x5557e6269960) _finish_write discard buffer(0x5557e9f00d80 space 0x5557e9f09278 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09358 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9f09358 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09438 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9f09438 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09518 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9f09518 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f095f8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9f095f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f096d8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd7c20 space 0x5557e9f096d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f097b8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd7cb0 space 0x5557e9f097b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09898 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e9f09898 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09978 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd62d0 space 0x5557e9f09978 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09a58 in 0x5557e6269960) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9f09a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09b38 in 0x5557e6269960) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9f09b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09c18 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58630 space 0x5557e9f09c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09cf8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9f09cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09dd8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9f09dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09eb8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b59a70 space 0x5557e9f09eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f09f98 in 0x5557e6269960) _finish_write discard buffer(0x5557e9b599e0 space 0x5557e9f09f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.752+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 done 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.752+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10c00 [] 2019-12-07T19:33:58.752+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.752+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=70, last_complete=21'10, committed=1, applied=1)) v2 -- 0x5557e9d89200 con 0x5557e79cac00 2019-12-07T19:33:58.752+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.752+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.753+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 7 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 161716+0+0 (crc 0 0 0) 0x5557e9fcb000 con 0x5557e79cb800 2019-12-07T19:33:58.753+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.753+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 21 maybe_share_map con 0x5557e79cb800 v2:172.21.15.131:6822/37079 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.753+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e9d5bb00 seq 5 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.2s3_head 0x5557e75b4d20) get_onode oid 3#1:41df217a:::smithi13140796-39:head# key 0x83800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.753+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1121 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# = 0 2019-12-07T19:33:58.753+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 2 keys 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 2 keys = 0 2019-12-07T19:33:58.753+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.753+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 0x0~27000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 3#1:41df217a:::smithi13140796-39:head# 0x0~27000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~27000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9d5bb00 20 blobs 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate 0x27000/1000,27000,0 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x31e000~27000/1000,27000,0 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x31e000~27000] 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7668850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7668850 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7730 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e7668850) get_ref 0x0~2000 Blob(0x5557e7668850 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7730 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e7668850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7668850 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7730 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e75c7748 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d36310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d36310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3260 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9d36310) get_ref 0x0~2000 Blob(0x5557e9d36310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3260 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9d36310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9d36310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3260 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9cb3278 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d36380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d36380 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36460 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9d36380) get_ref 0x0~2000 Blob(0x5557e9d36380 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36460 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9d36380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9d36380 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36460 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9d36478 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba31f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba31f0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3260 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba31f0) get_ref 0x0~2000 Blob(0x5557e9ba31f0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3260 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba31f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9ba31f0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3260 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3278 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba32d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba32d0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3340 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba32d0) get_ref 0x0~2000 Blob(0x5557e9ba32d0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3340 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba32d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9ba32d0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3340 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3358 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba33b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba33b0 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3420 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba33b0) get_ref 0x0~2000 Blob(0x5557e9ba33b0 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3420 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba33b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9ba33b0 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3420 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3438 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3490 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4fc0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3490) get_ref 0x0~2000 Blob(0x5557e9ba3490 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd4fc0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9ba3490 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd4fc0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9dd4fd8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd50a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ae00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd50a0 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ae00 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9dd50a0) get_ref 0x0~2000 Blob(0x5557e9dd50a0 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ae00 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9dd50a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9dd50a0 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5ae00 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9e5ae18 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c370a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2bce0 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c370a0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9c2bce0) get_ref 0x0~2000 Blob(0x5557e9c2bce0 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c370a0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9c2bce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9c2bce0 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c370a0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9c370b8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3500 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3570 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3500) get_ref 0x0~2000 Blob(0x5557e9ba3500 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3570 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9ba3500 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3570 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3588 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba35e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba35e0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3650 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba35e0) get_ref 0x0~2000 Blob(0x5557e9ba35e0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3650 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba35e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9ba35e0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3650 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3668 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba36c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba36c0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3730 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba36c0) get_ref 0x0~2000 Blob(0x5557e9ba36c0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3730 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba36c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9ba36c0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3730 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3748 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba37a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba37a0 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3810 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba37a0) get_ref 0x0~2000 Blob(0x5557e9ba37a0 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3810 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba37a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9ba37a0 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3810 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3828 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba38f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3880 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba38f0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3880) get_ref 0x0~2000 Blob(0x5557e9ba3880 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba38f0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9ba3880 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba38f0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3908 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba39d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3960 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba39d0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3960) get_ref 0x0~2000 Blob(0x5557e9ba3960 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba39d0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9ba3960 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba39d0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba39e8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3a40 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3ab0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3a40) get_ref 0x0~2000 Blob(0x5557e9ba3a40 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3ab0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9ba3a40 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3ab0 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3ac8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3b20 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3b90 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3b20) get_ref 0x0~2000 Blob(0x5557e9ba3b20 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3b90 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9ba3b20 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3b90 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3ba8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3c00 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3c70 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3c00) get_ref 0x0~2000 Blob(0x5557e9ba3c00 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3c70 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9ba3c00 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3c70 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3c88 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3ce0 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3d50 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3ce0) get_ref 0x0~2000 Blob(0x5557e9ba3ce0 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3d50 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9ba3ce0 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba3d50 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3d68 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3dc0 blob([0x344000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3e30 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3dc0) get_ref 0x0~1000 Blob(0x5557e9ba3dc0 blob([0x344000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3e30 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ba3dc0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~1000: 0x0~1000 Blob(0x5557e9ba3dc0 blob([0x344000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9ba3e30 sbid 0x0)) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ba3e48 in 0x5557e62695e0) _discard 0x0~1000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x27000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9b7dc88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 0x0~27000 = 0 2019-12-07T19:33:58.753+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.753+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5bb00 cost 246174 (21 ios * 4000 + 162174 bytes) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5bb00 onodes 0x5557e9b7db80 shared_blobs 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9b7dc88) update 3#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9b7dc88) update inline shard 380 bytes from 20 extents 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:41df217a:::smithi13140796-39:head# is 799 (413 bytes onode + 2 bytes spanning blobs + 384 bytes inline extents) 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5bb00 allocated 0x[31e000~27000] released 0x[] 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 freelist allocate 0x31e000~27000 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 prepare 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9d5bb00 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.753+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85760 finish 2019-12-07T19:33:58.753+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.753+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 aio_wait 2019-12-07T19:33:58.753+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5bb00 2019-12-07T19:33:58.753+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 io_done 2019-12-07T19:33:58.753+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.753+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.753+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.753+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2325 < min_flush_size 524288 2019-12-07T19:33:58.753+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7db80 had 1 2019-12-07T19:33:58.753+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.753+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2348 < min_flush_size 524288 2019-12-07T19:33:58.753+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2348 < min_flush_size 524288 2019-12-07T19:33:58.753+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.753+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4d0e4~92c to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.753+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4d0e4 0x4d0e4~92c to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.753+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.753+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1d0e4 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe4 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa10 and padding block with 0x5f0 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4da10 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000168205s (0.000042422s flush + 0.000125783s kv commit) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5bb00> 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.754+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 kv_submitted 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5bb00 2019-12-07T19:33:58.754+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 finishing 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5bb00 onodes 0x5557e9b7db80 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c7748 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6e10 space 0x5557e75c7748 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3278 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6fc0 space 0x5557e9ba3278 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3358 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6a20 space 0x5557e9ba3358 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3438 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6360 space 0x5557e9ba3438 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3588 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9e44480 space 0x5557e9ba3588 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3668 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e9ba3668 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3748 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9ba3748 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3828 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9ba3828 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3908 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9ba3908 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba39e8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9ba39e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3ac8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7c20 space 0x5557e9ba3ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3ba8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7cb0 space 0x5557e9ba3ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3c88 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f002d0 space 0x5557e9ba3c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 -- 0x5557e794ee00 con 0x5557e79cb800 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3d68 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f003f0 space 0x5557e9ba3d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba3e48 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00480 space 0x5557e9ba3e48 0x0~1000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c370b8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9e44510 space 0x5557e9c370b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb3278 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6ea0 space 0x5557e9cb3278 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36478 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6f30 space 0x5557e9d36478 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd4fd8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9e45b90 space 0x5557e9dd4fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5ae18 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9e445a0 space 0x5557e9e5ae18 0x0~2000 writing nocache) 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5bb00 done 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:58.754+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5bb00 [] 2019-12-07T19:33:58.754+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.754+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.754+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 98 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=71, reqid=client.4540.0:122, at_version=21'11, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e7a0e000 con 0x5557e79cac00 2019-12-07T19:33:58.754+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=71, reqid=client.4540.0:122, at_version=21'11, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.754+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=71, reqid=client.4540.0:122, at_version=21'11, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.754+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.754+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=71, reqid=client.4540.0:122, at_version=21'11, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.754+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.754+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=71, reqid=client.4540.0:122, at_version=21'11, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.754+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=71, reqid=client.4540.0:122, at_version=21'11, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.8s5( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.754+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=71, reqid=client.4540.0:122, at_version=21'11, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.754+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'10], crt=21'8) [21'11 (21'9) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:122 2019-12-07T19:33:58.741144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.754+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'10 crt=21'8 lcod 21'10 active mbc={}] add_log_entry 21'11 (21'9) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:122 2019-12-07T19:33:58.741144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.754+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'10 crt=21'9 lcod 21'10 active mbc={}] rollforward: entry=21'9 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:121 2019-12-07T19:33:58.741112+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.754+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'10 crt=21'9 lcod 21'10 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.754+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'10 crt=21'9 lcod 21'10 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.754+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.754+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.754+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9b10f00 seq 21 2019-12-07T19:33:58.754+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 3 keys 2019-12-07T19:33:58.754+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 3 keys = 0 2019-12-07T19:33:58.754+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.754+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.754+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.754+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.754+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10f00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10f00 onodes 0x5557ea011180 shared_blobs 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011288) update 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:14fc029d:::smithi13140796-41:head# is 893 (485 bytes onode + 2 bytes spanning blobs + 406 bytes inline extents) 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.754+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 prepare 2019-12-07T19:33:58.754+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 2019-12-07T19:33:58.754+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 io_done 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.754+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'10 crt=21'9 lcod 21'10 active mbc={}] op_applied version 21'11 2019-12-07T19:33:58.754+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.754+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1933 < min_flush_size 524288 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea011180 had 1 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1956 < min_flush_size 524288 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1956 < min_flush_size 524288 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4da10~7a4 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4da10 0x4da10~7a4 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1da10 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa10 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x1b4 and padding block with 0xe4c 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4e1b4 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.754+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000117426s (0.000000061s flush + 0.000117365s kv commit) 2019-12-07T19:33:58.754+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10f00> 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.754+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10f00 2019-12-07T19:33:58.754+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 finishing 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10f00 onodes 0x5557ea011180 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 done 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.754+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10f00 [] 2019-12-07T19:33:58.754+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.754+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.754+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.754+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=71, last_complete=21'11, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cac00 2019-12-07T19:33:58.754+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.755+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 99 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=72, reqid=client.4540.0:125, at_version=21'12, trim_to=0'0, roll_forward_to=21'10)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9c7a800 con 0x5557e79cac00 2019-12-07T19:33:58.755+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85ce0 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=72, reqid=client.4540.0:125, at_version=21'12, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.755+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=72, reqid=client.4540.0:125, at_version=21'12, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.755+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.755+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=72, reqid=client.4540.0:125, at_version=21'12, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.755+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.755+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=72, reqid=client.4540.0:125, at_version=21'12, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.755+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 prio 127 cost 0 latency 0.000074 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=72, reqid=client.4540.0:125, at_version=21'12, trim_to=0'0, roll_forward_to=21'10)) v2 pg pg[1.8s5( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'9 active mbc={}] 2019-12-07T19:33:58.755+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'9 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=72, reqid=client.4540.0:125, at_version=21'12, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.755+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log log((0'0,21'11], crt=21'9) [21'12 (21'10) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:125 2019-12-07T19:33:58.745945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.755+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'11 crt=21'9 lcod 21'11 active mbc={}] add_log_entry 21'12 (21'10) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:125 2019-12-07T19:33:58.745945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.755+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'11 crt=21'10 lcod 21'11 active mbc={}] rollforward: entry=21'10 (0'0) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:124 2019-12-07T19:33:58.745795+0000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.755+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'11 crt=21'10 lcod 21'11 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.756+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'11 crt=21'10 lcod 21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.756+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.756+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.756+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7693500 seq 22 2019-12-07T19:33:58.756+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 3 keys 2019-12-07T19:33:58.756+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 3 keys = 0 2019-12-07T19:33:58.756+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.756+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.756+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.756+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.756+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693500 onodes 0x5557ea011400 shared_blobs 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011508) update 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:10f769a3:::smithi13140796-42:head# is 909 (485 bytes onode + 2 bytes spanning blobs + 422 bytes inline extents) 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.756+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 prepare 2019-12-07T19:33:58.756+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693500 2019-12-07T19:33:58.756+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 io_done 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.756+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'11 crt=21'10 lcod 21'11 active mbc={}] op_applied version 21'12 2019-12-07T19:33:58.756+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 finish 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1949 < min_flush_size 524288 2019-12-07T19:33:58.756+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea011400 had 1 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1972 < min_flush_size 524288 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1972 < min_flush_size 524288 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4e1b4~7b4 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4e1b4 0x4e1b4~7b4 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1e1b4 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x1b4 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x968 and padding block with 0x698 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4e968 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.756+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000105308s (0.000000039s flush + 0.000105269s kv commit) 2019-12-07T19:33:58.756+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.756+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.756+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693500> 2019-12-07T19:33:58.756+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.756+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 kv_submitted 2019-12-07T19:33:58.756+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693500 2019-12-07T19:33:58.756+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693500 finishing 2019-12-07T19:33:58.756+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693500 onodes 0x5557ea011400 2019-12-07T19:33:58.756+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693500 done 2019-12-07T19:33:58.756+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.756+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693500 [] 2019-12-07T19:33:58.756+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.756+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.756+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.756+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=72, last_complete=21'12, committed=1, applied=1)) v2 -- 0x5557e9d89200 con 0x5557e79cac00 2019-12-07T19:33:58.756+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.758+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 8 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 182196+0+0 (crc 0 0 0) 0x5557e9d80800 con 0x5557e79cb800 2019-12-07T19:33:58.758+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.758+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.2s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] add_log_entry 21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.758+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e9b10c00 seq 6 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.2s3_head 0x5557e75b4d20) get_onode oid 3#1:4986b721:::smithi13140796-43:head# key 0x8380000000000000014986b7'!!smithi13140796-43!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.758+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1122 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# = 0 2019-12-07T19:33:58.758+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 2 keys 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 2 keys = 0 2019-12-07T19:33:58.758+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.758+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 0x0~2c000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 3#1:4986b721:::smithi13140796-43:head# 0x0~2c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~2c000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b10c00 22 blobs 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2c000/1000,2c000,0 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x345000~2c000/1000,2c000,0 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x345000~2c000] 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c76c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c76c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7b20 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e75c76c0) get_ref 0x0~2000 Blob(0x5557e75c76c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e75c7b20 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e75c76c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e75c76c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7b20 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e75c7b38 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed0770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed07e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed0770 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed07e0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0770) get_ref 0x0~2000 Blob(0x5557e9ed0770 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed07e0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9ed0770 blob([0x347000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed07e0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed07f8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed08c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed08c0 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0930 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed08c0) get_ref 0x0~2000 Blob(0x5557e9ed08c0 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0930 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed08c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9ed08c0 blob([0x349000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed0930 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed0948 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed09a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed09a0 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0a10 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed09a0) get_ref 0x0~2000 Blob(0x5557e9ed09a0 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0a10 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed09a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9ed09a0 blob([0x34b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed0a10 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed0a28 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed0a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed0a80 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0af0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0a80) get_ref 0x0~2000 Blob(0x5557e9ed0a80 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0af0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9ed0a80 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed0af0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed0b08 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed0b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed0b60 blob([0x34f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0bd0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0b60) get_ref 0x0~2000 Blob(0x5557e9ed0b60 blob([0x34f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0bd0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9ed0b60 blob([0x34f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed0bd0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed0be8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed0c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed0c40 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0cb0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0c40) get_ref 0x0~2000 Blob(0x5557e9ed0c40 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0cb0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9ed0c40 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed0cb0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed0cc8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed0d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed0d20 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0d90 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0d20) get_ref 0x0~2000 Blob(0x5557e9ed0d20 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0d90 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9ed0d20 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed0d90 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed0da8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed0e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed0e00 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0e70 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0e00) get_ref 0x0~2000 Blob(0x5557e9ed0e00 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0e70 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9ed0e00 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed0e70 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed0e88 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed0ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed0ee0 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0f50 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0ee0) get_ref 0x0~2000 Blob(0x5557e9ed0ee0 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed0f50 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9ed0ee0 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed0f50 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed0f68 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed0fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed0fc0 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1030 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0fc0) get_ref 0x0~2000 Blob(0x5557e9ed0fc0 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1030 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed0fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9ed0fc0 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed1030 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed1048 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed10a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed10a0 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1110 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed10a0) get_ref 0x0~2000 Blob(0x5557e9ed10a0 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1110 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed10a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9ed10a0 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed1110 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed1128 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed1180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed11f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed1180 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed11f0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1180) get_ref 0x0~2000 Blob(0x5557e9ed1180 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed11f0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9ed1180 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed11f0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed1208 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed1260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed12d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed1260 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed12d0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1260) get_ref 0x0~2000 Blob(0x5557e9ed1260 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed12d0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9ed1260 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed12d0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed12e8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed1340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed13b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed1340 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed13b0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1340) get_ref 0x0~2000 Blob(0x5557e9ed1340 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed13b0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9ed1340 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed13b0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed13c8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb35e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb35e0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3570 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9cb35e0) get_ref 0x0~2000 Blob(0x5557e9cb35e0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb3570 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9cb35e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9cb35e0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb3570 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9cb3588 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed1420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed1420 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1490 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1420) get_ref 0x0~2000 Blob(0x5557e9ed1420 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1490 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9ed1420 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed1490 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed14a8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed1500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed1500 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1570 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1500) get_ref 0x0~2000 Blob(0x5557e9ed1500 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1570 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9ed1500 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed1570 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed1588 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed15e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed15e0 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1650 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed15e0) get_ref 0x0~2000 Blob(0x5557e9ed15e0 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1650 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed15e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9ed15e0 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed1650 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed1668 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed16c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed16c0 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1730 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed16c0) get_ref 0x0~2000 Blob(0x5557e9ed16c0 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1730 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed16c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e9ed16c0 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed1730 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed1748 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed17a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed17a0 blob([0x36d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1810 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed17a0) get_ref 0x0~2000 Blob(0x5557e9ed17a0 blob([0x36d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed1810 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed17a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x5557e9ed17a0 blob([0x36d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed1810 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed1828 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ed1880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed18f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ed1880 blob([0x36f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed18f0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1880) get_ref 0x0~2000 Blob(0x5557e9ed1880 blob([0x36f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ed18f0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ed1880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x5557e9ed1880 blob([0x36f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ed18f0 sbid 0x0)) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ed1908 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x2c000 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.extentmap(0x5557ea011a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 0x0~2c000 = 0 2019-12-07T19:33:58.758+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.758+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.758+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10c00 cost 273846 (23 ios * 4000 + 181846 bytes) 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10c00 onodes 0x5557ea011900 shared_blobs 2019-12-07T19:33:58.758+0000 7f2b754dc700 20 bluestore.extentmap(0x5557ea011a08) update 3#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.759+0000 7f2b754dc700 20 bluestore.extentmap(0x5557ea011a08) update inline shard 421 bytes from 22 extents 2019-12-07T19:33:58.759+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:4986b721:::smithi13140796-43:head# is 840 (413 bytes onode + 2 bytes spanning blobs + 425 bytes inline extents) 2019-12-07T19:33:58.759+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10c00 allocated 0x[345000~2c000] released 0x[] 2019-12-07T19:33:58.759+0000 7f2b754dc700 10 freelist allocate 0x345000~2c000 2019-12-07T19:33:58.759+0000 7f2b754dc700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:33:58.759+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 prepare 2019-12-07T19:33:58.759+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b10c00 2019-12-07T19:33:58.759+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 100 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=74, reqid=client.4540.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c7a000 con 0x5557e79cac00 2019-12-07T19:33:58.759+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85a20 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=74, reqid=client.4540.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.759+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=74, reqid=client.4540.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.759+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.759+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d84160 finish 2019-12-07T19:33:58.759+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.759+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=74, reqid=client.4540.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.759+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=74, reqid=client.4540.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.759+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85a20 prio 127 cost 0 latency 0.000358 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=74, reqid=client.4540.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 pg pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'10 active mbc={}] 2019-12-07T19:33:58.759+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'10 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=74, reqid=client.4540.0:125, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.759+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log log((0'0,21'12], crt=21'10) [] 2019-12-07T19:33:58.759+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] rollforward: entry=21'11 (21'9) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:122 2019-12-07T19:33:58.741144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.759+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] rollforward: entry=21'12 (21'10) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:125 2019-12-07T19:33:58.745945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.759+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.759+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.759+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.759+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.759+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.759+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e7693200 seq 23 2019-12-07T19:33:58.759+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.759+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.759+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.759+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.759+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.759+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.759+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693200 onodes shared_blobs 2019-12-07T19:33:58.759+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.759+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 prepare 2019-12-07T19:33:58.759+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693200 2019-12-07T19:33:58.759+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 io_done 2019-12-07T19:33:58.759+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e9d85a20 finish 2019-12-07T19:33:58.759+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4e968~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4e968 0x4e968~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1e968 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x968 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xdd9 and padding block with 0x227 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4edd9 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.759+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.759+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.001330871s (0.000000084s flush + 0.001330787s kv commit) 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.761+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 aio_wait 2019-12-07T19:33:58.761+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10c00 2019-12-07T19:33:58.761+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 io_done 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693200> 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.761+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 kv_submitted 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693200 2019-12-07T19:33:58.761+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 finishing 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693200 onodes 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693200 done 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.761+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693200 [] 2019-12-07T19:33:58.761+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.761+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.761+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=74, last_complete=21'12, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cac00 2019-12-07T19:33:58.761+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1552 < min_flush_size 524288 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea011900 had 1 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1575 < min_flush_size 524288 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1575 < min_flush_size 524288 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4edd9~627 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4edd9 0x4edd9~627 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1edd9 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xdd9 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x400 and padding block with 0xc00 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4f400 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.761+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000268220s (0.000143076s flush + 0.000125144s kv commit) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.761+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10c00> 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.761+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10c00 2019-12-07T19:33:58.761+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 finishing 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10c00 onodes 0x5557ea011900 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e75c7b38 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b59e60 space 0x5557e75c7b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb3588 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f007e0 space 0x5557e9cb3588 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed07f8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e9ed07f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed0948 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd62d0 space 0x5557e9ed0948 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed0a28 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00510 space 0x5557e9ed0a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed0b08 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00000 space 0x5557e9ed0b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed0be8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00090 space 0x5557e9ed0be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed0cc8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00870 space 0x5557e9ed0cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed0da8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00900 space 0x5557e9ed0da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 -- 0x5557e758c200 con 0x5557e79cb800 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed0e88 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00990 space 0x5557e9ed0e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed0f68 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00a20 space 0x5557e9ed0f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed1048 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00ab0 space 0x5557e9ed1048 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed1128 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b58bd0 space 0x5557e9ed1128 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed1208 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00c60 space 0x5557e9ed1208 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed12e8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00cf0 space 0x5557e9ed12e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed13c8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00d80 space 0x5557e9ed13c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed14a8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00120 space 0x5557e9ed14a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed1588 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f001b0 space 0x5557e9ed1588 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed1668 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00360 space 0x5557e9ed1668 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed1748 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00240 space 0x5557e9ed1748 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed1828 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00e10 space 0x5557e9ed1828 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ed1908 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f00ea0 space 0x5557e9ed1908 0x0~2000 writing nocache) 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 done 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:58.761+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10c00 [] 2019-12-07T19:33:58.761+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.761+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.763+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 9 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 190388+0+0 (crc 0 0 0) 0x5557e9d80800 con 0x5557e79cb800 2019-12-07T19:33:58.763+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d846e0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.763+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d846e0 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.2s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'2], crt=0'0) [21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.763+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e9b11500 seq 7 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.2s3_head 0x5557e75b4d20) get_onode oid 3#1:41fe8a11:::smithi13140796-45:head# key 0x83800000000000000141fe8a1121736d'ithi13140796-45!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.763+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1123 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# = 0 2019-12-07T19:33:58.763+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 2 keys 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 2 keys = 0 2019-12-07T19:33:58.763+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.763+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.763+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 3#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.763+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~2e000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b11500 23 blobs 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2e000/1000,2e000,0 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x371000~2e000/1000,2e000,0 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x371000~2e000] 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c77a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff08c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c77a0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff08c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e75c77a0) get_ref 0x0~2000 Blob(0x5557e75c77a0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff08c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e75c77a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e75c77a0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff08c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff08d8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff0310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff0310 blob([0x373000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0380 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0310) get_ref 0x0~2000 Blob(0x5557e9ff0310 blob([0x373000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0380 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9ff0310 blob([0x373000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0380 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff0398 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff0460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff04d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff0460 blob([0x375000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff04d0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0460) get_ref 0x0~2000 Blob(0x5557e9ff0460 blob([0x375000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff04d0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9ff0460 blob([0x375000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff04d0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff04e8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff0540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff05b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff0540 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff05b0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0540) get_ref 0x0~2000 Blob(0x5557e9ff0540 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff05b0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9ff0540 blob([0x377000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff05b0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff05c8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff0620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff0620 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0690 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0620) get_ref 0x0~2000 Blob(0x5557e9ff0620 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0690 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9ff0620 blob([0x379000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0690 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff06a8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff0700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff0700 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0770 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0700) get_ref 0x0~2000 Blob(0x5557e9ff0700 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0770 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9ff0700 blob([0x37b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0770 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff0788 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff07e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff07e0 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0850 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff07e0) get_ref 0x0~2000 Blob(0x5557e9ff07e0 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0850 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff07e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9ff07e0 blob([0x37d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0850 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff0868 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff12d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff12d0 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1340 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff12d0) get_ref 0x0~2000 Blob(0x5557e9ff12d0 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1340 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff12d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9ff12d0 blob([0x37f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1340 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1358 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff13b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff13b0 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1420 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff13b0) get_ref 0x0~2000 Blob(0x5557e9ff13b0 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1420 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff13b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9ff13b0 blob([0x381000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1420 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1438 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1490 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1500 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1490) get_ref 0x0~2000 Blob(0x5557e9ff1490 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1500 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9ff1490 blob([0x383000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1500 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1518 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff15e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1570 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff15e0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1570) get_ref 0x0~2000 Blob(0x5557e9ff1570 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff15e0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9ff1570 blob([0x385000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff15e0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff15f8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff16c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1650 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff16c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1650) get_ref 0x0~2000 Blob(0x5557e9ff1650 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff16c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9ff1650 blob([0x387000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff16c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff16d8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff17a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1730 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff17a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1730) get_ref 0x0~2000 Blob(0x5557e9ff1730 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff17a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9ff1730 blob([0x389000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff17a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff17b8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1810 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1880 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1810) get_ref 0x0~2000 Blob(0x5557e9ff1810 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1880 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9ff1810 blob([0x38b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1880 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1898 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff18f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff18f0 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1960 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff18f0) get_ref 0x0~2000 Blob(0x5557e9ff18f0 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1960 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff18f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9ff18f0 blob([0x38d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1960 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1978 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff19d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff19d0 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1a40 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff19d0) get_ref 0x0~2000 Blob(0x5557e9ff19d0 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1a40 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff19d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9ff19d0 blob([0x38f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1a40 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1a58 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1ab0 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1b20 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1ab0) get_ref 0x0~2000 Blob(0x5557e9ff1ab0 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1b20 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9ff1ab0 blob([0x391000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1b20 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1b38 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1b90 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1c00 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1b90) get_ref 0x0~2000 Blob(0x5557e9ff1b90 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1c00 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9ff1b90 blob([0x393000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1c00 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1c18 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1c70 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1ce0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1c70) get_ref 0x0~2000 Blob(0x5557e9ff1c70 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1ce0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9ff1c70 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1ce0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1cf8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1d50 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1dc0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1d50) get_ref 0x0~2000 Blob(0x5557e9ff1d50 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1dc0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e9ff1d50 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1dc0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1dd8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1e30 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1ea0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1e30) get_ref 0x0~2000 Blob(0x5557e9ff1e30 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1ea0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x5557e9ff1e30 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1ea0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1eb8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff1f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff1f10 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1f80 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1f10) get_ref 0x0~2000 Blob(0x5557e9ff1f10 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff1f80 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff1f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x5557e9ff1f10 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff1f80 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff1f98 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff2000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff2000 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2070 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2000) get_ref 0x0~2000 Blob(0x5557e9ff2000 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2070 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x5557e9ff2000 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2070 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff2088 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x2e000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9eb5508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 = 0 2019-12-07T19:33:58.764+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.764+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11500 cost 286038 (24 ios * 4000 + 190038 bytes) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11500 onodes 0x5557e9eb5400 shared_blobs 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9eb5508) update 3#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9eb5508) update inline shard 440 bytes from 23 extents 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:41fe8a11:::smithi13140796-45:head# is 859 (413 bytes onode + 2 bytes spanning blobs + 444 bytes inline extents) 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11500 allocated 0x[371000~2e000] released 0x[] 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 freelist allocate 0x371000~2e000 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 freelist _xor first_key 0x300000 last_key 0x380000 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 prepare 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b11500 2019-12-07T19:33:58.764+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 10 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9fcb000 con 0x5557e79cb800 2019-12-07T19:33:58.764+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d84840 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.764+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.764+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.764+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.764+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d846e0 finish 2019-12-07T19:33:58.764+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.764+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 aio_wait 2019-12-07T19:33:58.764+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11500 2019-12-07T19:33:58.764+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 io_done 2019-12-07T19:33:58.764+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.764+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.764+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1600 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e9d84840 prio 127 cost 0 latency 0.000326 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.2s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb5400 had 1 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1623 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1623 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4f400~657 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4f400 0x4f400~657 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=0'0 lcod 21'2 active mbc={}] append_log log((0'0,21'3], crt=0'0) [21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1f400 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x400 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa57 and padding block with 0x5a9 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=0'0 lcod 21'2 active mbc={}] add_log_entry 21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] rollforward: entry=21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x4fa57 2019-12-07T19:33:58.765+0000 7f2b794e4700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000134270s (0.000040633s flush + 0.000093637s kv commit) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11500> 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 kv_submitted 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11500 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 finishing 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11500 onodes 0x5557e9eb5400 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff0398 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f010e0 space 0x5557e9ff0398 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff04e8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01170 space 0x5557e9ff04e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff05c8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01200 space 0x5557e9ff05c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e7121500 seq 8 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff06a8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01290 space 0x5557e9ff06a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff0788 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01320 space 0x5557e9ff0788 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff0868 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f013b0 space 0x5557e9ff0868 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff08d8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01050 space 0x5557e9ff08d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1358 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01440 space 0x5557e9ff1358 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1438 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f014d0 space 0x5557e9ff1438 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1518 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9ff1518 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff15f8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b58630 space 0x5557e9ff15f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 3 keys 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff16d8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b58990 space 0x5557e9ff16d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff17b8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b58900 space 0x5557e9ff17b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1898 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b59a70 space 0x5557e9ff1898 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1978 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b599e0 space 0x5557e9ff1978 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1a58 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b59e60 space 0x5557e9ff1a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1b38 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9e44480 space 0x5557e9ff1b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 3 keys = 0 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1c18 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9e44510 space 0x5557e9ff1c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1cf8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9e45b90 space 0x5557e9ff1cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1dd8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9e445a0 space 0x5557e9ff1dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1eb8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e79aabd0 space 0x5557e9ff1eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff1f98 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6e10 space 0x5557e9ff1f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff2088 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6fc0 space 0x5557e9ff2088 0x0~2000 writing nocache) 2019-12-07T19:33:58.765+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11500 done 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 prepare 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11500 [] 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121500 onodes 0x5557ea011900 shared_blobs 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore.extentmap(0x5557ea011a08) update 3#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:4986b721:::smithi13140796-43:head# is 912 (485 bytes onode + 2 bytes spanning blobs + 425 bytes inline extents) 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 prepare 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121500 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 io_done 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1449 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1963 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e9d84840 finish 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea011900 had 1 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1986 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e794ee00 con 0x5557e79cb800 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1986 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x4fa57~7c2 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x4fa57 0x4fa57~7c2 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.708867+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _allocate len 0x219 from 0 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _allocate len 0x219 from 1 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,acd180000 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd180000~10000/10000,10000,acd180000 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x60000 to include allocated 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 27 (was clean) 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x1fa57 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa57 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x219 and padding block with 0xde7 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x60000 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x50219 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (27) on file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]), flushing log 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 27 len 0x49 crc 0x1d7e2c1d) 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x1a000~1000 to file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x1a000 0x1a000~1000 to file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1a000 2019-12-07T19:33:58.765+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 11 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9ba1000 con 0x5557e79cb800 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x1b000 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:58.765+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.765+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'2 lcod 21'3 active mbc={}] 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'2 lcod 21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'2 lcod 21'3 active mbc={}] append_log log((0'0,21'4], crt=21'2) [21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] add_log_entry 21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.765+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e9d5bb00 seq 9 2019-12-07T19:33:58.765+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 3 keys 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 3 keys = 0 2019-12-07T19:33:58.765+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.765+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5bb00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5bb00 onodes 0x5557e9b7db80 shared_blobs 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9b7dc88) update 3#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:41df217a:::smithi13140796-39:head# is 871 (485 bytes onode + 2 bytes spanning blobs + 384 bytes inline extents) 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5bb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 prepare 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5bb00 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 io_done 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.765+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85760 finish 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000211 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 27 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000540978s (0.000000075s flush + 0.000540903s kv commit) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121500> 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 kv_submitted 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121500 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1911 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121500 finishing 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121500 onodes 0x5557ea011900 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121500 done 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5bb00 kv_queued 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121500 [] 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7db80 had 1 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.765+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 -- 0x5557e758c200 con 0x5557e79cb800 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1934 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1934 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x50219~78e to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x50219 0x50219~78e to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x20219 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x219 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x9a7 and padding block with 0x659 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x509a7 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.765+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000092945s (0.000000061s flush + 0.000092884s kv commit) 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.765+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5bb00> 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 kv_submitted 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5bb00 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 finishing 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5bb00 onodes 0x5557e9b7db80 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5bb00 done 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5bb00 [] 2019-12-07T19:33:58.765+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.765+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.765+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.765+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e9d89800 con 0x5557e79cb800 2019-12-07T19:33:58.765+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.765+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 msgr2=0x5557e7a96580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 94 2019-12-07T19:33:58.765+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 msgr2=0x5557e7a96580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.765+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 0x5557e7a96580 crc :-1 s=READY pgs=15 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.766+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 0x5557e7a96580 crc :-1 s=READY pgs=15 cs=0 l=0 rx=0 tx=0)._fault initiating reconnect 2019-12-07T19:33:58.766+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 0x5557e7a96580 unknown :-1 s=BANNER_CONNECTING pgs=15 cs=1 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.766+0000 7f2b834f8700 10 osd.6 21 ms_handle_connect con 0x5557e79cb800 2019-12-07T19:33:58.766+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] >> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] conn(0x5557e79cb800 0x5557e7a96580 crc :-1 s=READY pgs=15 cs=1 l=0 rx=0 tx=0).ready entity=osd.0 client_cookie=66df44187c63ea59 server_cookie=2175ee4a8507a6a8 in_seq=11 out_seq=10 2019-12-07T19:33:58.769+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 12 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e79e2000 con 0x5557e79cb800 2019-12-07T19:33:58.769+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.769+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.2s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'5], crt=21'2) [21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'2 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'3 lcod 21'5 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'3 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'3 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.769+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e9d5a300 seq 10 2019-12-07T19:33:58.769+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 3 keys 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 3 keys = 0 2019-12-07T19:33:58.769+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.769+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.769+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5a300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5a300 onodes 0x5557e9eb5400 shared_blobs 2019-12-07T19:33:58.769+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9eb5508) update 3#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:41fe8a11:::smithi13140796-45:head# is 931 (485 bytes onode + 2 bytes spanning blobs + 444 bytes inline extents) 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5a300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.770+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 prepare 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5a300 2019-12-07T19:33:58.770+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 io_done 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.770+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'3 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.770+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d84160 finish 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1971 < min_flush_size 524288 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb5400 had 1 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1994 < min_flush_size 524288 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1994 < min_flush_size 524288 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x509a7~7ca to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x509a7 0x509a7~7ca to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x209a7 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x9a7 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x171 and padding block with 0xe8f 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x51171 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.770+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000088001s (0.000000059s flush + 0.000087942s kv commit) 2019-12-07T19:33:58.770+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.770+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.770+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5a300> 2019-12-07T19:33:58.770+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.770+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 kv_submitted 2019-12-07T19:33:58.770+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5a300 2019-12-07T19:33:58.770+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a300 finishing 2019-12-07T19:33:58.770+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5a300 onodes 0x5557e9eb5400 2019-12-07T19:33:58.770+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a300 done 2019-12-07T19:33:58.770+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:58.770+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5a300 [] 2019-12-07T19:33:58.770+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.770+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.770+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e794ee00 con 0x5557e79cb800 2019-12-07T19:33:58.770+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.770+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 13 ==== MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e7557b80 con 0x5557e79cb800 2019-12-07T19:33:58.770+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.770+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.770+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000050 MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.770+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.770+0000 7f2b754dc700 25 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.770+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 0x0~1000 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x2c000 (180224) 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c76c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7b20 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e75c76c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7b20 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.770+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.771+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e75c76c0 blob([0x345000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7b20 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.771+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e75c7b38 in 0x5557e62695e0) _discard 0x0~1000 2019-12-07T19:33:58.771+0000 7f2b754dc700 20 _add level 1 near 0 on buffer(0x5557e9cd6a20 space 0x5557e75c7b38 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.771+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 0x0~1000 = 4096 2019-12-07T19:33:58.771+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.771+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.771+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.771+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.771+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] get_hash_info: found on disk, size 180224 2019-12-07T19:33:58.771+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# hinfo_key 2019-12-07T19:33:58.771+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.2s3_head 3#1:4986b721:::smithi13140796-43:head# hinfo_key = 0 2019-12-07T19:33:58.771+0000 7f2b754dc700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 -- 0x5557e795d680 con 0x5557e79cb800 2019-12-07T19:33:58.771+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85340 finish 2019-12-07T19:33:58.771+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.771+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 14 ==== MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9ff8000 con 0x5557e79cb800 2019-12-07T19:33:58.771+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e7695600 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.771+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e7695600 prio 127 cost 0 latency 0.000047 MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.771+0000 7f2b794e4700 25 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.771+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 0x0~1000 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x27000 (159744) 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7668850 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7730 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e7668850 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7730 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e7668850 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c7730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e75c7748 in 0x5557e62695e0) _discard 0x0~1000 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 _add level 1 near 0 on buffer(0x5557e9f00f30 space 0x5557e75c7748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 0x0~1000 = 4096 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] get_hash_info: found on disk, size 159744 2019-12-07T19:33:58.771+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# hinfo_key 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.2s3_head 3#1:41df217a:::smithi13140796-39:head# hinfo_key = 0 2019-12-07T19:33:58.771+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 -- 0x5557ea011b80 con 0x5557e79cb800 2019-12-07T19:33:58.771+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e7695600 finish 2019-12-07T19:33:58.771+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.776+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 66 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2 ==== 198580+0+0 (crc 0 0 0) 0x5557e9c7b800 con 0x5557e79ca800 2019-12-07T19:33:58.776+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85080 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.776+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d85080 prio 127 cost 0 latency 0.000052 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2 pg pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=40, reqid=client.4540.0:139, at_version=21'11, trim_to=0'0, roll_forward_to=21'10)) v2 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log log((0'0,21'10], crt=21'10) [21'11 (0'0) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:139 2019-12-07T19:33:58.773780+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'10 crt=21'10 lcod 21'10 active mbc={}] add_log_entry 21'11 (0'0) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:139 2019-12-07T19:33:58.773780+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'10 crt=21'10 lcod 21'10 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'10 crt=21'10 lcod 21'10 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.776+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7a93800 seq 20 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) get_onode oid 1#1:6ee4dfe7:::smithi13140796-47:head# key 0x8180000000000000016ee4dfe721736d'ithi13140796-47!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.776+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1124 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# = 0 2019-12-07T19:33:58.776+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 2 keys 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 2 keys = 0 2019-12-07T19:33:58.776+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.776+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 0x0~30000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:6ee4dfe7:::smithi13140796-47:head# 0x0~30000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~30000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a93800 24 blobs 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x30000/1000,30000,0 2019-12-07T19:33:58.776+0000 7f2b734d8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x39f000~30000/1000,30000,0 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x39f000~30000] 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37e30 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b730 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37e30) get_ref 0x0~2000 Blob(0x5557e9d37e30 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b730 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9d37e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9d37e30 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b730 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9e5b748 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29500 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29570 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29500) get_ref 0x0~2000 Blob(0x5557e9f29500 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29570 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9f29500 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f29570 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f29588 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f295e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29490 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f295e0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29490) get_ref 0x0~2000 Blob(0x5557e9f29490 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f295e0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9f29490 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f295e0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f295f8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f296c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29650 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f296c0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29650) get_ref 0x0~2000 Blob(0x5557e9f29650 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f296c0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9f29650 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f296c0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f296d8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f297a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29730 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f297a0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29730) get_ref 0x0~2000 Blob(0x5557e9f29730 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f297a0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9f29730 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f297a0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f297b8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29810 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29880 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29810) get_ref 0x0~2000 Blob(0x5557e9f29810 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29880 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9f29810 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f29880 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f29898 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f298f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f298f0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29960 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f298f0) get_ref 0x0~2000 Blob(0x5557e9f298f0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29960 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f298f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9f298f0 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f29960 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f29978 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f299d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f299d0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29a40 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f299d0) get_ref 0x0~2000 Blob(0x5557e9f299d0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29a40 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f299d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9f299d0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f29a40 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f29a58 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29ab0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29b20 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29ab0) get_ref 0x0~2000 Blob(0x5557e9f29ab0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29b20 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9f29ab0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f29b20 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f29b38 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29b90 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29c00 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29b90) get_ref 0x0~2000 Blob(0x5557e9f29b90 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29c00 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9f29b90 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f29c00 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f29c18 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29c70 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29ce0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29c70) get_ref 0x0~2000 Blob(0x5557e9f29c70 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29ce0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9f29c70 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f29ce0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f29cf8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f29d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f29d50 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29dc0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29d50) get_ref 0x0~2000 Blob(0x5557e9f29d50 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f29dc0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9f29d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9f29d50 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f29dc0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9f29dd8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3490 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3500 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3490) get_ref 0x0~2000 Blob(0x5557e9ff3490 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3500 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9ff3490 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3500 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff3518 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff35e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff35e0 blob([0x3b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3650 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff35e0) get_ref 0x0~2000 Blob(0x5557e9ff35e0 blob([0x3b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3650 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff35e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9ff35e0 blob([0x3b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3650 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff3668 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff36c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff36c0 blob([0x3bb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2af0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff36c0) get_ref 0x0~2000 Blob(0x5557e9ff36c0 blob([0x3bb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2af0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff36c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9ff36c0 blob([0x3bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2af0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff2b08 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff2bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff2bd0 blob([0x3bd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2b60 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff2bd0) get_ref 0x0~2000 Blob(0x5557e9ff2bd0 blob([0x3bd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2b60 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff2bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9ff2bd0 blob([0x3bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2b60 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff2b78 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff37a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3730 blob([0x3bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff37a0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3730) get_ref 0x0~2000 Blob(0x5557e9ff3730 blob([0x3bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff37a0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e9ff3730 blob([0x3bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff37a0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff37b8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3810 blob([0x3c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3880 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3810) get_ref 0x0~2000 Blob(0x5557e9ff3810 blob([0x3c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3880 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9ff3810 blob([0x3c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3880 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff3898 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff38f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff38f0 blob([0x3c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3960 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff38f0) get_ref 0x0~2000 Blob(0x5557e9ff38f0 blob([0x3c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3960 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff38f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9ff38f0 blob([0x3c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3960 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff3978 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff39d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff39d0 blob([0x3c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3a40 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff39d0) get_ref 0x0~2000 Blob(0x5557e9ff39d0 blob([0x3c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3a40 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff39d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e9ff39d0 blob([0x3c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3a40 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff3a58 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3ab0 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3b20 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3ab0) get_ref 0x0~2000 Blob(0x5557e9ff3ab0 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3b20 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x5557e9ff3ab0 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3b20 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff3b38 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3b90 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3c00 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3b90) get_ref 0x0~2000 Blob(0x5557e9ff3b90 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3c00 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x5557e9ff3b90 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3c00 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff3c18 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3c70 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3ce0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3c70) get_ref 0x0~2000 Blob(0x5557e9ff3c70 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3ce0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x5557e9ff3c70 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3ce0 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff3cf8 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3d50 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0c40 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3d50) get_ref 0x0~2000 Blob(0x5557e9ff3d50 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0c40 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.blob(0x5557e9ff3d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x5557e9ff3d50 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0c40 sbid 0x0)) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9ff0c58 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x30000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb5008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 0x0~30000 = 0 2019-12-07T19:33:58.777+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.777+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93800 cost 298230 (25 ios * 4000 + 198230 bytes) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93800 onodes 0x5557e9eb4f00 shared_blobs 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb5008) update 1#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb5008) update inline shard 459 bytes from 24 extents 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:6ee4dfe7:::smithi13140796-47:head# is 878 (413 bytes onode + 2 bytes spanning blobs + 463 bytes inline extents) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93800 allocated 0x[39f000~30000] released 0x[] 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 freelist allocate 0x39f000~30000 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 prepare 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a93800 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'10 crt=21'10 lcod 21'10 active mbc={}] op_applied version 21'11 2019-12-07T19:33:58.777+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d85080 finish 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.777+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 aio_wait 2019-12-07T19:33:58.777+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93800 2019-12-07T19:33:58.777+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 io_done 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1590 < min_flush_size 524288 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4f00 had 1 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1613 < min_flush_size 524288 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1613 < min_flush_size 524288 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x51171~64d to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x51171 0x51171~64d to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x21171 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x171 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x7be and padding block with 0x842 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x517be 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.777+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000112193s (0.000034308s flush + 0.000077885s kv commit) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.777+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a93800> 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.777+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 kv_submitted 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93800 2019-12-07T19:33:58.777+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 finishing 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93800 onodes 0x5557e9eb4f00 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b748 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e458c0 space 0x5557e9e5b748 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f29588 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e45950 space 0x5557e9f29588 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f295f8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e459e0 space 0x5557e9f295f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f296d8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e45a70 space 0x5557e9f296d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f297b8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e45b00 space 0x5557e9f297b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f29898 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e45cb0 space 0x5557e9f29898 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f29978 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e45d40 space 0x5557e9f29978 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f29a58 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e45dd0 space 0x5557e9f29a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f29b38 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9e45e60 space 0x5557e9f29b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f29c18 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00510 space 0x5557e9f29c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=40, last_complete=21'11, committed=1, applied=1)) v2 -- 0x5557e9d89600 con 0x5557e79ca800 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f29cf8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00000 space 0x5557e9f29cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f29dd8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00090 space 0x5557e9f29dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff0c58 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00120 space 0x5557e9ff0c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff2b08 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00990 space 0x5557e9ff2b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff2b78 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00a20 space 0x5557e9ff2b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3518 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00870 space 0x5557e9ff3518 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3668 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00900 space 0x5557e9ff3668 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff37b8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00ab0 space 0x5557e9ff37b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3898 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00c60 space 0x5557e9ff3898 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3978 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd62d0 space 0x5557e9ff3978 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3a58 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6240 space 0x5557e9ff3a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3b38 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9cd6990 space 0x5557e9ff3b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3c18 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00cf0 space 0x5557e9ff3c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3cf8 in 0x5557e6268fc0) _finish_write discard buffer(0x5557e9f00d80 space 0x5557e9ff3cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93800 done 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.777+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93800 [] 2019-12-07T19:33:58.777+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.777+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.778+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 15 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 108468+0+0 (crc 0 0 0) 0x5557e9c79000 con 0x5557e79cb800 2019-12-07T19:33:58.778+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.778+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'6], crt=21'3) [21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'3 lcod 21'6 active mbc={}] add_log_entry 21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] rollforward: entry=21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] rollforward: entry=21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.778+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e9d5b500 seq 11 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.2s3_head 0x5557e75b4d20) get_onode oid 3#1:4c9e444c:::smithi13140796-48:head# key 0x8380000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.778+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1125 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# = 0 2019-12-07T19:33:58.778+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 2 keys 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 2 keys = 0 2019-12-07T19:33:58.778+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.778+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 0x0~1a000 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 3#1:4c9e444c:::smithi13140796-48:head# 0x0~1a000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.778+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1a000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9d5b500 13 blobs 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.778+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1a000/1000,1a000,0 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x3cf000~1a000/1000,1a000,0 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x3cf000~1a000] 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff0b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff0b60 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3570 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0b60) get_ref 0x0~2000 Blob(0x5557e9ff0b60 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3570 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9ff0b60 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3570 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff3588 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff33b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff33b0 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3340 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff33b0) get_ref 0x0~2000 Blob(0x5557e9ff33b0 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3340 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff33b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9ff33b0 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3340 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff3358 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff31f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3260 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff31f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff3260) get_ref 0x0~2000 Blob(0x5557e9ff3260 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff31f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff3260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9ff3260 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff31f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff3208 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3180 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3110 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff3180) get_ref 0x0~2000 Blob(0x5557e9ff3180 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3110 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff3180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9ff3180 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3110 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff3128 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff30a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff30a0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3030 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff30a0) get_ref 0x0~2000 Blob(0x5557e9ff30a0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff3030 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff30a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9ff30a0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3030 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff3048 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff2fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff2fc0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2f50 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2fc0) get_ref 0x0~2000 Blob(0x5557e9ff2fc0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2f50 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9ff2fc0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2f50 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff2f68 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff2ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff2ee0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2e70 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2ee0) get_ref 0x0~2000 Blob(0x5557e9ff2ee0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2e70 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9ff2ee0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2e70 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff2e88 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff2e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff2e00 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2d90 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2e00) get_ref 0x0~2000 Blob(0x5557e9ff2e00 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2d90 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9ff2e00 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2d90 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff2da8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff2d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff2d20 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2cb0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2d20) get_ref 0x0~2000 Blob(0x5557e9ff2d20 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff2cb0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9ff2d20 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2cb0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff2cc8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff2c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff03f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff2c40 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff03f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2c40) get_ref 0x0~2000 Blob(0x5557e9ff2c40 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff03f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff2c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9ff2c40 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff03f0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff0408 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff09a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff09a0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0af0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff09a0) get_ref 0x0~2000 Blob(0x5557e9ff09a0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0af0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff09a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9ff09a0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0af0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff0b08 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff0930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7669730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff0930 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669730 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0930) get_ref 0x0~2000 Blob(0x5557e9ff0930 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669730 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9ff0930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9ff0930 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7669730 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e7669748 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d921c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d921c0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29500 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9d921c0) get_ref 0x0~2000 Blob(0x5557e9d921c0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a29500 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.blob(0x5557e9d921c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9d921c0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7a29500 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e7a29518 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1a000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9c7d788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 0x0~1a000 = 0 2019-12-07T19:33:58.779+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.779+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5b500 cost 164118 (14 ios * 4000 + 108118 bytes) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5b500 onodes 0x5557e9c7d680 shared_blobs 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9c7d788) update 3#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9c7d788) update inline shard 250 bytes from 13 extents 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:4c9e444c:::smithi13140796-48:head# is 669 (413 bytes onode + 2 bytes spanning blobs + 254 bytes inline extents) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5b500 allocated 0x[3cf000~1a000] released 0x[] 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 freelist allocate 0x3cf000~1a000 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 prepare 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9d5b500 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.779+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.779+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 aio_wait 2019-12-07T19:33:58.779+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5b500 2019-12-07T19:33:58.779+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 io_done 2019-12-07T19:33:58.779+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 16 ==== MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557ea011b80 con 0x5557e79cb800 2019-12-07T19:33:58.779+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d84f20 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.779+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e9d84f20 prio 127 cost 0 latency 0.000050 MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.779+0000 7f2b794e4700 25 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.779+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 0x0~1000 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x2e000 (188416) 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e75c77a0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff08c0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e75c77a0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff08c0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1381 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7d680 had 1 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1404 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1404 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x517be~57c to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x517be 0x517be~57c to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x217be 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x7be 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xd3a and padding block with 0x2c6 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x51d3a 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.779+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000194239s (0.000111635s flush + 0.000082604s kv commit) 2019-12-07T19:33:58.779+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5b500> 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.779+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 kv_submitted 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5b500 2019-12-07T19:33:58.779+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b500 finishing 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5b500 onodes 0x5557e9c7d680 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7669748 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f014d0 space 0x5557e7669748 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a29518 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f002d0 space 0x5557e7a29518 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff0408 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01050 space 0x5557e9ff0408 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff0b08 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01440 space 0x5557e9ff0b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff2cc8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f013b0 space 0x5557e9ff2cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff2da8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6f30 space 0x5557e9ff2da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff2e88 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6ea0 space 0x5557e9ff2e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff2f68 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7cb0 space 0x5557e9ff2f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3048 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7c20 space 0x5557e9ff3048 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3128 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7050 space 0x5557e9ff3128 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3208 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7320 space 0x5557e9ff3208 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3358 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6c60 space 0x5557e9ff3358 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff3588 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd6870 space 0x5557e9ff3588 0x0~2000 writing nocache) 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5b500 done 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:58.779+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5b500 [] 2019-12-07T19:33:58.779+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.779+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e75c77a0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff08c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff08d8 in 0x5557e62695e0) _discard 0x0~1000 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 _add level 1 near 0 on buffer(0x5557e9b58630 space 0x5557e9ff08d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 0x0~1000 = 4096 2019-12-07T19:33:58.779+0000 7f2b794e4700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 lcod 21'6 active mbc={}] get_hash_info: found on disk, size 188416 2019-12-07T19:33:58.779+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# hinfo_key 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.2s3_head 3#1:41fe8a11:::smithi13140796-45:head# hinfo_key = 0 2019-12-07T19:33:58.779+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 -- 0x5557e9ff8000 con 0x5557e79cb800 2019-12-07T19:33:58.779+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e9d84f20 finish 2019-12-07T19:33:58.780+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e9d89800 con 0x5557e79cb800 2019-12-07T19:33:58.780+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.784+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 67 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c78000 con 0x5557e79ca800 2019-12-07T19:33:58.784+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.784+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.784+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.784+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.784+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.784+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.784+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 pg pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] 2019-12-07T19:33:58.784+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=41, reqid=client.4540.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'10 active mbc={}] append_log log((0'0,21'11], crt=21'10) [] 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'11 (0'0) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:139 2019-12-07T19:33:58.773780+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.785+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7a92c00 seq 21 2019-12-07T19:33:58.785+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92c00 onodes shared_blobs 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 prepare 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92c00 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 io_done 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d85600 finish 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x51d3a~471 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x51d3a 0x51d3a~471 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x21d3a 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xd3a 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x1ab and padding block with 0xe55 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x521ab 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000082124s (0.000000084s flush + 0.000082040s kv commit) 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a92c00> 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.785+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 kv_submitted 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92c00 2019-12-07T19:33:58.785+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92c00 finishing 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92c00 onodes 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92c00 done 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.785+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92c00 [] 2019-12-07T19:33:58.785+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.785+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=41, last_complete=21'11, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79ca800 2019-12-07T19:33:58.785+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 68 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e9c79800 con 0x5557e79ca800 2019-12-07T19:33:58.785+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d854a0 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.785+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d854a0 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 pg pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=42, reqid=client.4540.0:140, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log log((0'0,21'11], crt=21'11) [21'12 (21'11) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:140 2019-12-07T19:33:58.773812+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] add_log_entry 21'12 (21'11) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:140 2019-12-07T19:33:58.773812+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.785+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7121800 seq 22 2019-12-07T19:33:58.785+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 3 keys 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 3 keys = 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7121800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7121800 onodes 0x5557e9eb4f00 shared_blobs 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9eb5008) update 1#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:6ee4dfe7:::smithi13140796-47:head# is 950 (485 bytes onode + 2 bytes spanning blobs + 463 bytes inline extents) 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7121800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 prepare 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7121800 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 io_done 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] op_applied version 21'12 2019-12-07T19:33:58.785+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e9d854a0 finish 2019-12-07T19:33:58.785+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1990 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4f00 had 1 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2013 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2013 < min_flush_size 524288 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x521ab~7dd to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x521ab 0x521ab~7dd to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x221ab 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x1ab 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x988 and padding block with 0x678 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x52988 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.785+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000094636s (0.000000061s flush + 0.000094575s kv commit) 2019-12-07T19:33:58.785+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7121800> 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.785+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 kv_submitted 2019-12-07T19:33:58.785+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7121800 2019-12-07T19:33:58.786+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7121800 finishing 2019-12-07T19:33:58.786+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7121800 onodes 0x5557e9eb4f00 2019-12-07T19:33:58.786+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7121800 done 2019-12-07T19:33:58.786+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.786+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7121800 [] 2019-12-07T19:33:58.786+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.786+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.786+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=42, last_complete=21'12, committed=1, applied=1)) v2 -- 0x5557e9d88e00 con 0x5557e79ca800 2019-12-07T19:33:58.786+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.786+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 17 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c7a800 con 0x5557e79cb800 2019-12-07T19:33:58.788+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d84dc0 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.788+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 prio 127 cost 0 latency 0.000047 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'7], crt=21'5) [] 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.788+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e9d5af00 seq 12 2019-12-07T19:33:58.788+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5af00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5af00 onodes shared_blobs 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5af00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 prepare 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5af00 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 io_done 2019-12-07T19:33:58.788+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 finish 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x52988~471 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x52988 0x52988~471 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x22988 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x988 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.788+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 18 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 96536+0+0 (crc 0 0 0) 0x5557e7a0e000 con 0x5557e79cb800 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xdf9 and padding block with 0x207 2019-12-07T19:33:58.788+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d84f20 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.788+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 19 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e79e2000 con 0x5557e79cb800 2019-12-07T19:33:58.788+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x52df9 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e9d84f20 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000091658s (0.000000070s flush + 0.000091588s kv commit) 2019-12-07T19:33:58.788+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.788+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5af00> 2019-12-07T19:33:58.788+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.788+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 kv_submitted 2019-12-07T19:33:58.788+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5af00 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.788+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 finishing 2019-12-07T19:33:58.788+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5af00 onodes 2019-12-07T19:33:58.788+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5af00 done 2019-12-07T19:33:58.788+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:58.788+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5af00 [] 2019-12-07T19:33:58.788+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.788+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.788+0000 7f2b794e4700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e7630000 seq 13 2019-12-07T19:33:58.788+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 2 keys 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 2 keys = 0 2019-12-07T19:33:58.788+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.788+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.788+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 0x1a000~17000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 3#1:4c9e444c:::smithi13140796-48:head# 0x1a000~17000 - have 0x1a000 (106496) bytes fadvise_flags 0x0 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x1a000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7630000 12 blobs 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 fbmap_alloc 0x5557e6282e00 allocate 0x17000/1000,17000,0 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x3e9000~17000/1000,17000,0 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x3e9000~17000] 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7452230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74521c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7452230 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74521c0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452230) get_ref 0x0~2000 Blob(0x5557e7452230 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74521c0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e7452230 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74521c0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74521d8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74522a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7452310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74522a0 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452310 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74522a0) get_ref 0x0~2000 Blob(0x5557e74522a0 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452310 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74522a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e74522a0 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452310 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452328 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74523f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7452150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74523f0 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452150 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74523f0) get_ref 0x0~2000 Blob(0x5557e74523f0 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452150 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74523f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e74523f0 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452150 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452168 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74520e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7452070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74520e0 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452070 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74520e0) get_ref 0x0~2000 Blob(0x5557e74520e0 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452070 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74520e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e74520e0 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452070 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452088 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7452460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74524d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7452460 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74524d0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452460) get_ref 0x0~2000 Blob(0x5557e7452460 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74524d0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e7452460 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74524d0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74524e8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7452540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74525b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7452540 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74525b0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452540) get_ref 0x0~2000 Blob(0x5557e7452540 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74525b0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e7452540 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74525b0 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74525c8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7452620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7452690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7452620 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452690 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452620) get_ref 0x0~2000 Blob(0x5557e7452620 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452690 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e7452620 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452690 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74526a8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ff3420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ff3420 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0d20 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff3420) get_ref 0x0~2000 Blob(0x5557e9ff3420 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ff0d20 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff3420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x5557e9ff3420 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0d20 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff0d38 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7452700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7452770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7452700 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452770 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452700) get_ref 0x0~2000 Blob(0x5557e7452700 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452770 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x5557e7452700 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452770 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452788 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74527e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7452850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74527e0 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452850 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74527e0) get_ref 0x0~2000 Blob(0x5557e74527e0 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452850 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74527e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x5557e74527e0 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452850 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452868 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74528c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7452930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74528c0 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452930 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74528c0) get_ref 0x0~2000 Blob(0x5557e74528c0 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452930 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74528c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x5557e74528c0 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452930 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452948 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74529a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7452a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74529a0 blob([0x3ff000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452a10 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74529a0) get_ref 0x0~1000 Blob(0x5557e74529a0 blob([0x3ff000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7452a10 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.blob(0x5557e74529a0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x30000~1000: 0x0~1000 Blob(0x5557e74529a0 blob([0x3ff000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7452a10 sbid 0x0)) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452a28 in 0x5557e62695e0) _discard 0x0~1000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x31000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.extentmap(0x5557e9c7d788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 0x1a000~17000 = 0 2019-12-07T19:33:58.788+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.788+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630000 cost 148186 (13 ios * 4000 + 96186 bytes) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630000 onodes 0x5557e9c7d680 shared_blobs 2019-12-07T19:33:58.788+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 69 ==== MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9c78800 con 0x5557e79ca800 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.extentmap(0x5557e9c7d788) update 3#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.788+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84580 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore.extentmap(0x5557e9c7d788) update inline shard 475 bytes from 25 extents 2019-12-07T19:33:58.788+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:4c9e444c:::smithi13140796-48:head# is 894 (413 bytes onode + 2 bytes spanning blobs + 479 bytes inline extents) 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630000 allocated 0x[3e9000~17000] released 0x[] 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 freelist allocate 0x3e9000~17000 2019-12-07T19:33:58.788+0000 7f2b794e4700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.788+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 70 ==== MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9d58000 con 0x5557e79ca800 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 prepare 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7630000 2019-12-07T19:33:58.788+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.788+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e9d84580 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 pg pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] _handle_message: MOSDECSubOpWrite(1.6s1 21/18 ECSubWrite(tid=43, reqid=client.4540.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log log((0'0,21'12], crt=21'11) [] 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] rollforward: entry=21'12 (21'11) modify 1:6ee4dfe7:::smithi13140796-47:head by client.4540.0:140 2019-12-07T19:33:58.773812+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.788+0000 7f2b774e0700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557e7693b00 seq 23 2019-12-07T19:33:58.788+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693b00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693b00 onodes shared_blobs 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693b00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 prepare 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693b00 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 io_done 2019-12-07T19:33:58.788+0000 7f2b774e0700 10 osd.6 21 dequeue_op 0x5557e9d84580 finish 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.788+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e769c000 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x52df9~471 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x52df9 0x52df9~471 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.788+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4f20 prio 127 cost 0 latency 0.000125 MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x22df9 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xdf9 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x26a and padding block with 0xd96 2019-12-07T19:33:58.788+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] _handle_message: MOSDECSubOpRead(1.6s1 21/18 ECSubRead(tid=44, to_read={1:6ee4dfe7:::smithi13140796-47:head=0,4096,0}, subchunks={1:6ee4dfe7:::smithi13140796-47:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.788+0000 7f2b734d8700 25 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.788+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 0x0~1000 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x30000 (196608) 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9d37e30 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b730 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9d37e30 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b730 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.788+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.788+0000 7f2b794e4700 10 osd.6 21 dequeue_op 0x5557e9d84f20 finish 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x5326a 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.788+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e758c200 con 0x5557e79cb800 2019-12-07T19:33:58.788+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.788+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.789+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85340 prio 127 cost 0 latency 0.000613 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.2s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693b00> 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000163462s (0.000000080s flush + 0.000163382s kv commit) 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 kv_submitted 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693b00 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 finishing 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693b00 onodes 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693b00 done 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] append_log log((0'0,21'8], crt=21'7) [21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693b00 [] 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.789+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.789+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 aio_wait 2019-12-07T19:33:58.789+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630000 2019-12-07T19:33:58.789+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9d37e30 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.789+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e9e5b748 in 0x5557e6268fc0) _discard 0x0~1000 2019-12-07T19:33:58.789+0000 7f2b734d8700 20 _add level 1 near 0 on buffer(0x5557e9f001b0 space 0x5557e9e5b748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.789+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 io_done 2019-12-07T19:33:58.789+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 0x0~1000 = 4096 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e7692c00 seq 14 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.789+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 3 keys 2019-12-07T19:33:58.789+0000 7f2b734d8700 20 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 3 keys = 0 2019-12-07T19:33:58.789+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.789+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] get_hash_info: Getting attr on 1:6ee4dfe7:::smithi13140796-47:head 2019-12-07T19:33:58.789+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.789+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] get_hash_info: not in cache 1:6ee4dfe7:::smithi13140796-47:head 2019-12-07T19:33:58.789+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.789+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] get_hash_info: found on disk, size 196608 2019-12-07T19:33:58.789+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.789+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.6s1_head 1#1:6ee4dfe7:::smithi13140796-47:head# hinfo_key = 0 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692c00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692c00 onodes 0x5557e9c7d680 shared_blobs 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore.extentmap(0x5557e9c7d788) update 3#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.789+0000 7f2b734d8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.6s0 21/18 ECSubReadReply(tid=44, attrs_read=0)) v2 -- 0x5557e76ca500 con 0x5557e79ca800 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:4c9e444c:::smithi13140796-48:head# is 966 (485 bytes onode + 2 bytes spanning blobs + 479 bytes inline extents) 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2034 < min_flush_size 524288 2019-12-07T19:33:58.789+0000 7f2b734d8700 10 osd.6 21 dequeue_op 0x5557e75c4f20 finish 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 prepare 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692c00 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 io_done 2019-12-07T19:33:58.789+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.789+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 21/18 ECSubWriteReply(tid=43, last_complete=21'12, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79ca800 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7d680 had 2 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 2 2019-12-07T19:33:58.789+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85340 finish 2019-12-07T19:33:58.789+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2057 < min_flush_size 524288 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2057 < min_flush_size 524288 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x5326a~809 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x5326a 0x5326a~809 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2326a 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x26a 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa73 and padding block with 0x58d 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x53a73 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630000> 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000178528s (0.000033153s flush + 0.000145375s kv commit) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 kv_submitted 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630000 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 finishing 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630000 onodes 0x5557e9c7d680 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7452088 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9b58990 space 0x5557e7452088 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7452168 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f019e0 space 0x5557e7452168 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2006 < min_flush_size 524288 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74521d8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9cd7680 space 0x5557e74521d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7452328 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01950 space 0x5557e7452328 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 -- 0x5557e9d88c00 con 0x5557e79cb800 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74524e8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01a70 space 0x5557e74524e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74525c8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01b00 space 0x5557e74525c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74526a8 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01b90 space 0x5557e74526a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7452788 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01cb0 space 0x5557e7452788 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7d680 had 1 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7452868 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01d40 space 0x5557e7452868 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.789+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7452948 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01dd0 space 0x5557e7452948 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2029 < min_flush_size 524288 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2029 < min_flush_size 524288 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7452a28 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01e60 space 0x5557e7452a28 0x0~1000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ff0d38 in 0x5557e62695e0) _finish_write discard buffer(0x5557e9f01c20 space 0x5557e9ff0d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x53a73~7ed to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630000 done 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692c00 kv_submitted 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630000 [] 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x53a73 0x53a73~7ed to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x23a73 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa73 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x260 and padding block with 0xda0 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x54260 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.789+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000109932s (0.000000053s flush + 0.000109879s kv commit) 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.789+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692c00> 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 kv_submitted 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692c00 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692c00 finishing 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692c00 onodes 0x5557e9c7d680 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692c00 done 2019-12-07T19:33:58.789+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692c00 [] 2019-12-07T19:33:58.789+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.789+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.789+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 -- 0x5557e9d89c00 con 0x5557e79cb800 2019-12-07T19:33:58.789+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.792+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 20 ==== MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9ba1000 con 0x5557e79cb800 2019-12-07T19:33:58.792+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d846e0 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.792+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d846e0 prio 127 cost 0 latency 0.000068 MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'9], crt=21'7) [] 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] rollforward: entry=21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] rollforward: entry=21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.792+0000 7f2b754dc700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e7692000 seq 15 2019-12-07T19:33:58.792+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692000 onodes shared_blobs 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 prepare 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692000 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 io_done 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d846e0 finish 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x54260~471 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x54260 0x54260~471 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x24260 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x260 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x6d1 and padding block with 0x92f 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x546d1 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.792+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000111311s (0.000000062s flush + 0.000111249s kv commit) 2019-12-07T19:33:58.792+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.792+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.792+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692000> 2019-12-07T19:33:58.792+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.792+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 kv_submitted 2019-12-07T19:33:58.792+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692000 2019-12-07T19:33:58.792+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692000 finishing 2019-12-07T19:33:58.792+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692000 onodes 2019-12-07T19:33:58.792+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692000 done 2019-12-07T19:33:58.792+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:33:58.792+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692000 [] 2019-12-07T19:33:58.792+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.792+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.792+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 -- 0x5557e794ee00 con 0x5557e79cb800 2019-12-07T19:33:58.792+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.792+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 21 ==== MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9ff8000 con 0x5557e79cb800 2019-12-07T19:33:58.792+0000 7f2b967b4700 15 osd.6 21 enqueue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.792+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e767f000 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85760 prio 127 cost 0 latency 0.000054 MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] _handle_message: MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.792+0000 7f2b754dc700 25 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.792+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 0x0~1000 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x31000 (200704) 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9ff0b60 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3570 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9ff0b60 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3570 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9ff0b60 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3570 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 bluestore.BufferSpace(0x5557e9ff3588 in 0x5557e62695e0) _discard 0x0~1000 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 _add level 1 near 0 on buffer(0x5557e9f01ef0 space 0x5557e9ff3588 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.792+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 0x0~1000 = 4096 2019-12-07T19:33:58.792+0000 7f2b754dc700 20 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.793+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] get_hash_info: Getting attr on 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.793+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] get_hash_info: not in cache 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.793+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.793+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] get_hash_info: found on disk, size 200704 2019-12-07T19:33:58.793+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# hinfo_key 2019-12-07T19:33:58.793+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# hinfo_key = 0 2019-12-07T19:33:58.793+0000 7f2b754dc700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 -- 0x5557ea011680 con 0x5557e79cb800 2019-12-07T19:33:58.793+0000 7f2b754dc700 10 osd.6 21 dequeue_op 0x5557e9d85760 finish 2019-12-07T19:33:58.793+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.793+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 48 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 112564+0+0 (crc 0 0 0) 0x5557e9d83800 con 0x5557e79cbc00 2019-12-07T19:33:58.793+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4580 prio 127 cost 0 latency 0.000041 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.793+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.793+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.793+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c4580 prio 127 cost 0 latency 0.000088 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.793+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.793+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.793+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.793+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.793+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.793+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.793+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7630600 seq 13 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.bs1_head 0x5557e75b41e0) get_onode oid 1#1:d313ddd1:::smithi13140796-49:head# key 0x818000000000000001d313ddd121736d'ithi13140796-49!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.794+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1126 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# = 0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2 keys 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2 keys = 0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~1b000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:d313ddd1:::smithi13140796-49:head# 0x0~1b000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1b000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7630600 14 blobs 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1b000/1000,1b000,0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x400000~1b000/1000,1b000,0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x400000~1b000] 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d108c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37ab0 blob([0x400000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d108c0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d37ab0) get_ref 0x0~2000 Blob(0x5557e9d37ab0 blob([0x400000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d108c0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d37ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9d37ab0 blob([0x400000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d108c0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d108d8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2bf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2bf10 blob([0x402000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8460 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9c2bf10) get_ref 0x0~2000 Blob(0x5557e9c2bf10 blob([0x402000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d8460 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9c2bf10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9c2bf10 blob([0x402000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74d8460 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e74d8478 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb3880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb13b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb3880 blob([0x404000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb13b0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb3880) get_ref 0x0~2000 Blob(0x5557e9cb3880 blob([0x404000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb13b0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb3880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9cb3880 blob([0x404000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb13b0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb13c8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1340 blob([0x406000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1490 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb1340) get_ref 0x0~2000 Blob(0x5557e9cb1340 blob([0x406000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1490 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb1340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9cb1340 blob([0x406000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1490 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb14a8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1500 blob([0x408000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1570 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb1500) get_ref 0x0~2000 Blob(0x5557e9cb1500 blob([0x408000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1570 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb1500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9cb1500 blob([0x408000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1570 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb1588 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb15e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb15e0 blob([0x40a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1650 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb15e0) get_ref 0x0~2000 Blob(0x5557e9cb15e0 blob([0x40a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1650 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb15e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9cb15e0 blob([0x40a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1650 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb1668 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74531f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7453260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74531f0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453260 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74531f0) get_ref 0x0~2000 Blob(0x5557e74531f0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453260 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74531f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e74531f0 blob([0x40c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7453260 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e7453278 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74532d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7453340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74532d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453340 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74532d0) get_ref 0x0~2000 Blob(0x5557e74532d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453340 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74532d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e74532d0 blob([0x40e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7453340 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e7453358 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e74533b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7453420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e74533b0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453420 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74533b0) get_ref 0x0~2000 Blob(0x5557e74533b0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453420 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74533b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e74533b0 blob([0x410000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7453420 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e7453438 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7453490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7453500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7453490 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453500 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453490) get_ref 0x0~2000 Blob(0x5557e7453490 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453500 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e7453490 blob([0x412000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7453500 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e7453518 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7453570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74535e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7453570 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74535e0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453570) get_ref 0x0~2000 Blob(0x5557e7453570 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74535e0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e7453570 blob([0x414000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74535e0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e74535f8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7453650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74536c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7453650 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74536c0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453650) get_ref 0x0~2000 Blob(0x5557e7453650 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74536c0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e7453650 blob([0x416000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74536c0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e74536d8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7453730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74537a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7453730 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74537a0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453730) get_ref 0x0~2000 Blob(0x5557e7453730 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74537a0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e7453730 blob([0x418000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74537a0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e74537b8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7453810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7453880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7453810 blob([0x41a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453880 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453810) get_ref 0x0~1000 Blob(0x5557e7453810 blob([0x41a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453880 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7453810) get_ref init 0x1000, 1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~1000: 0x0~1000 Blob(0x5557e7453810 blob([0x41a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7453880 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e7453898 in 0x5557e6269c00) _discard 0x0~1000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1b000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~1b000 = 0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630600 cost 172214 (15 ios * 4000 + 112214 bytes) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630600 onodes 0x5557e9b95680 shared_blobs 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95788) update 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95788) update inline shard 266 bytes from 14 extents 2019-12-07T19:33:58.794+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 323883 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:d313ddd1:::smithi13140796-49:head# is 685 (413 bytes onode + 2 bytes spanning blobs + 270 bytes inline extents) 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630600 allocated 0x[400000~1b000] released 0x[] 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 freelist allocate 0x400000~1b000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630600 prepare 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7630600 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.794+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c4580 finish 2019-12-07T19:33:58.794+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.794+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630600 aio_wait 2019-12-07T19:33:58.794+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630600 2019-12-07T19:33:58.794+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630600 io_done 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1397 < min_flush_size 524288 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95680 had 1 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1420 < min_flush_size 524288 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1420 < min_flush_size 524288 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x546d1~58c to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x546d1 0x546d1~58c to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x246d1 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x6d1 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xc5d and padding block with 0x3a3 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x54c5d 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.794+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000115307s (0.000038420s flush + 0.000076887s kv commit) 2019-12-07T19:33:58.794+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630600> 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.794+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630600 kv_submitted 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630600 2019-12-07T19:33:58.794+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630600 finishing 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630600 onodes 0x5557e9b95680 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7453278 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9f01e60 space 0x5557e7453278 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7453358 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e000 space 0x5557e7453358 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7453438 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e090 space 0x5557e7453438 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7453518 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e120 space 0x5557e7453518 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74535f8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e1b0 space 0x5557e74535f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74536d8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e240 space 0x5557e74536d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74537b8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e2d0 space 0x5557e74537b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7453898 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e360 space 0x5557e7453898 0x0~1000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d8478 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44900 space 0x5557e74d8478 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=22, last_complete=21'5, committed=1, applied=1)) v2 -- 0x5557e9c76800 con 0x5557e79cbc00 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb13c8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44990 space 0x5557e9cb13c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb14a8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44a20 space 0x5557e9cb14a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1588 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44ab0 space 0x5557e9cb1588 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1668 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44b40 space 0x5557e9cb1668 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d108d8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e44870 space 0x5557e9d108d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630600 done 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.794+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630600 [] 2019-12-07T19:33:58.794+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.794+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.794+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.795+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.796+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 49 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 116660+0+0 (crc 0 0 0) 0x5557e9d82000 con 0x5557e79cbc00 2019-12-07T19:33:58.796+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.796+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.bs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] add_log_entry 21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7693b00 seq 14 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.bs1_head 0x5557e75b41e0) get_onode oid 1#1:dce63cc3:::smithi13140796-50:head# key 0x818000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.796+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1127 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# = 0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~1c000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:dce63cc3:::smithi13140796-50:head# 0x0~1c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1c000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7693b00 14 blobs 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1c000/1000,1c000,0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x41b000~1c000/1000,1c000,0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x41b000~1c000] 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb1730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d10e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb1730 blob([0x41b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d10e00 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1730) get_ref 0x0~2000 Blob(0x5557e9cb1730 blob([0x41b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d10e00 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9cb1730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e9cb1730 blob([0x41b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d10e00 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d10e18 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6a80 blob([0x41d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bb90 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6a80) get_ref 0x0~2000 Blob(0x5557e9eb6a80 blob([0x41d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c2bb90 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9eb6a80 blob([0x41d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2bb90 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9c2bba8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6a10 blob([0x41f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6af0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6a10) get_ref 0x0~2000 Blob(0x5557e9eb6a10 blob([0x41f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6af0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6a10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9eb6a10 blob([0x41f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb6af0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb6b08 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6b60 blob([0x421000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6bd0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6b60) get_ref 0x0~2000 Blob(0x5557e9eb6b60 blob([0x421000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6bd0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9eb6b60 blob([0x421000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb6bd0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb6be8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6c40 blob([0x423000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6cb0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6c40) get_ref 0x0~2000 Blob(0x5557e9eb6c40 blob([0x423000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6cb0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9eb6c40 blob([0x423000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb6cb0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb6cc8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6d20 blob([0x425000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6d90 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6d20) get_ref 0x0~2000 Blob(0x5557e9eb6d20 blob([0x425000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6d90 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9eb6d20 blob([0x425000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb6d90 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb6da8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6e00 blob([0x427000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6e70 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6e00) get_ref 0x0~2000 Blob(0x5557e9eb6e00 blob([0x427000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6e70 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9eb6e00 blob([0x427000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb6e70 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb6e88 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6ee0 blob([0x429000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6f50 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6ee0) get_ref 0x0~2000 Blob(0x5557e9eb6ee0 blob([0x429000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb6f50 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9eb6ee0 blob([0x429000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb6f50 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb6f68 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9eb6fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7453f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9eb6fc0 blob([0x42b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453f80 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6fc0) get_ref 0x0~2000 Blob(0x5557e9eb6fc0 blob([0x42b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453f80 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9eb6fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9eb6fc0 blob([0x42b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7453f80 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e7453f98 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea080000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea080000 blob([0x42d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080070 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080000) get_ref 0x0~2000 Blob(0x5557ea080000 blob([0x42d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080070 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557ea080000 blob([0x42d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080070 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080088 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea0800e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea0800e0 blob([0x42f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080150 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0800e0) get_ref 0x0~2000 Blob(0x5557ea0800e0 blob([0x42f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080150 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0800e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557ea0800e0 blob([0x42f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080150 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080168 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea0801c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea0801c0 blob([0x431000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080230 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0801c0) get_ref 0x0~2000 Blob(0x5557ea0801c0 blob([0x431000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080230 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0801c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557ea0801c0 blob([0x431000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080230 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080248 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea0802a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea0802a0 blob([0x433000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080310 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0802a0) get_ref 0x0~2000 Blob(0x5557ea0802a0 blob([0x433000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080310 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0802a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557ea0802a0 blob([0x433000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080310 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080328 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea080380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea0803f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea080380 blob([0x435000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea0803f0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080380) get_ref 0x0~2000 Blob(0x5557ea080380 blob([0x435000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea0803f0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557ea080380 blob([0x435000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea0803f0 sbid 0x0)) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080408 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1c000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9eb4388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~1c000 = 0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693b00 cost 176310 (15 ios * 4000 + 116310 bytes) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693b00 onodes 0x5557e9eb4280 shared_blobs 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9eb4388) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9eb4388) update inline shard 269 bytes from 14 extents 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 688 (413 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693b00 allocated 0x[41b000~1c000] released 0x[] 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 freelist allocate 0x41b000~1c000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 prepare 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7693b00 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.796+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c58c0 finish 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.796+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 aio_wait 2019-12-07T19:33:58.796+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693b00 2019-12-07T19:33:58.796+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 io_done 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1400 < min_flush_size 524288 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4280 had 1 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1423 < min_flush_size 524288 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1423 < min_flush_size 524288 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x54c5d~58f to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x54c5d 0x54c5d~58f to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x24c5d 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xc5d 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x1ec and padding block with 0xe14 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x551ec 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.796+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000139669s (0.000040842s flush + 0.000098827s kv commit) 2019-12-07T19:33:58.796+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7693b00> 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.796+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 kv_submitted 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693b00 2019-12-07T19:33:58.796+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693b00 finishing 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693b00 onodes 0x5557e9eb4280 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7453f98 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e5a0 space 0x5557e7453f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c2bba8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e45290 space 0x5557e9c2bba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d10e18 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e45200 space 0x5557e9d10e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb6b08 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e45320 space 0x5557e9eb6b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb6be8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e453b0 space 0x5557e9eb6be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb6cc8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e45440 space 0x5557e9eb6cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb6da8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e454d0 space 0x5557e9eb6da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb6e88 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e45560 space 0x5557e9eb6e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb6f68 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e455f0 space 0x5557e9eb6f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080088 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e630 space 0x5557ea080088 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=23, last_complete=21'6, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cbc00 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080168 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e6c0 space 0x5557ea080168 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080248 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e750 space 0x5557ea080248 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080328 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e7e0 space 0x5557ea080328 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080408 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e870 space 0x5557ea080408 0x0~2000 writing nocache) 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693b00 done 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.796+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693b00 [] 2019-12-07T19:33:58.796+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.796+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.796+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.797+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 50 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 1364+0+0 (crc 0 0 0) 0x5557e9d82800 con 0x5557e79cbc00 2019-12-07T19:33:58.797+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000034 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.797+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000061 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:154, at_version=21'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'6], crt=21'6) [21'7 (21'2) delete 1:c87708e9:::smithi13140796-17:head by client.4540.0:154 2019-12-07T19:33:58.792683+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 0, new_object: 0] 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] add_log_entry 21'7 (21'2) delete 1:c87708e9:::smithi13140796-17:head by client.4540.0:154 2019-12-07T19:33:58.792683+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.797+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7631200 seq 14 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.3s5_head 0x5557e75b4000) get_onode oid 5#1:c87708e9:::smithi13140796-17:head#7 key 0x858000000000000001c87708e921736d'ithi13140796-17!='0xfffffffffffffffe0000000000000007'o' 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.3s5_head 0x5557e75b4000) r -2 v.len 0 2019-12-07T19:33:58.797+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# -> 5#1:c87708e9:::smithi13140796-17:head#7 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head# -> 5#1:c87708e9:::smithi13140796-17:head#7 = 0 2019-12-07T19:33:58.797+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7631200 cost 5014 (1 ios * 4000 + 1014 bytes) 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7631200 onodes 0x5557e76caf00 shared_blobs 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e76cb008) update 5#1:c87708e9:::smithi13140796-17:head#7 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:c87708e9:::smithi13140796-17:head#7 is 662 (485 bytes onode + 2 bytes spanning blobs + 175 bytes inline extents) 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7631200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 prepare 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7631200 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 io_done 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.797+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c5b80 finish 2019-12-07T19:33:58.797+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.797+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1354 < min_flush_size 524288 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76caf00 had 1 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95900 had 1 2019-12-07T19:33:58.797+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1377 < min_flush_size 524288 2019-12-07T19:33:58.797+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1377 < min_flush_size 524288 2019-12-07T19:33:58.797+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.797+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x551ec~561 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.797+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x551ec 0x551ec~561 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x251ec 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x1ec 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.797+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x74d and padding block with 0x8b3 2019-12-07T19:33:58.798+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x5574d 2019-12-07T19:33:58.798+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.798+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.798+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.798+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.798+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000109834s (0.000000096s flush + 0.000109738s kv commit) 2019-12-07T19:33:58.798+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.798+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.798+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7631200> 2019-12-07T19:33:58.798+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.798+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 kv_submitted 2019-12-07T19:33:58.798+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7631200 2019-12-07T19:33:58.798+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7631200 finishing 2019-12-07T19:33:58.798+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7631200 onodes 0x5557e76caf00 2019-12-07T19:33:58.798+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7631200 done 2019-12-07T19:33:58.798+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.798+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7631200 [] 2019-12-07T19:33:58.798+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.798+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.798+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=24, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e9c76600 con 0x5557e79cbc00 2019-12-07T19:33:58.798+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.798+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.799+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 51 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 96536+0+0 (crc 0 0 0) 0x5557e9d81800 con 0x5557e79cbc00 2019-12-07T19:33:58.799+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.799+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000464 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.bs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'6], crt=21'4) [21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'4 lcod 21'6 active mbc={}] add_log_entry 21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7692f00 seq 15 2019-12-07T19:33:58.799+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2 keys 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2 keys = 0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x1b000~17000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:d313ddd1:::smithi13140796-49:head# 0x1b000~17000 - have 0x1b000 (110592) bytes fadvise_flags 0x0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x1b000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7692f00 12 blobs 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate 0x17000/1000,17000,0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x437000~17000/1000,17000,0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x437000~17000] 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d10d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb69a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d10d90 blob([0x437000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb69a0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d10d90) get_ref 0x0~2000 Blob(0x5557e9d10d90 blob([0x437000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9eb69a0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d10d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x5557e9d10d90 blob([0x437000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9eb69a0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9eb69b8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea080a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea080a80 blob([0x439000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080af0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080a80) get_ref 0x0~2000 Blob(0x5557ea080a80 blob([0x439000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080af0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x5557ea080a80 blob([0x439000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080af0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080b08 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea080bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea080bd0 blob([0x43b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080c40 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080bd0) get_ref 0x0~2000 Blob(0x5557ea080bd0 blob([0x43b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080c40 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x5557ea080bd0 blob([0x43b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080c40 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080c58 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea080cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea080cb0 blob([0x43d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080d20 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080cb0) get_ref 0x0~2000 Blob(0x5557ea080cb0 blob([0x43d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080d20 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x5557ea080cb0 blob([0x43d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080d20 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080d38 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea080d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea080d90 blob([0x43f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080e00 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080d90) get_ref 0x0~2000 Blob(0x5557ea080d90 blob([0x43f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080e00 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x5557ea080d90 blob([0x43f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080e00 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080e18 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea080e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea080e70 blob([0x441000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080ee0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080e70) get_ref 0x0~2000 Blob(0x5557ea080e70 blob([0x441000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080ee0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x5557ea080e70 blob([0x441000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080ee0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080ef8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea080f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea080fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea080f50 blob([0x443000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080fc0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080f50) get_ref 0x0~2000 Blob(0x5557ea080f50 blob([0x443000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea080fc0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea080f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x5557ea080f50 blob([0x443000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea080fc0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea080fd8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea081030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea0810a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea081030 blob([0x445000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea0810a0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea081030) get_ref 0x0~2000 Blob(0x5557ea081030 blob([0x445000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea0810a0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea081030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x5557ea081030 blob([0x445000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea0810a0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea0810b8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea081110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea081180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea081110 blob([0x447000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081180 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea081110) get_ref 0x0~2000 Blob(0x5557ea081110 blob([0x447000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081180 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea081110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x5557ea081110 blob([0x447000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea081180 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea081198 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea0811f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea081260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea0811f0 blob([0x449000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081260 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0811f0) get_ref 0x0~2000 Blob(0x5557ea0811f0 blob([0x449000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081260 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0811f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x5557ea0811f0 blob([0x449000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea081260 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea081278 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea0812d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea0813b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea0812d0 blob([0x44b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea0813b0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0812d0) get_ref 0x0~2000 Blob(0x5557ea0812d0 blob([0x44b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea0813b0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea0812d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x5557ea0812d0 blob([0x44b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea0813b0 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea0813c8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea081420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea081490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea081420 blob([!~1000,0x44d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081490 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea081420) get_ref 0x1000~1000 Blob(0x5557ea081420 blob([!~1000,0x44d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081490 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.blob(0x5557ea081420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x31000~1000: 0x1000~1000 Blob(0x5557ea081420 blob([!~1000,0x44d000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x5557ea081490 sbid 0x0)) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea0814a8 in 0x5557e6269c00) _discard 0x1000~1000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x32000 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9b95788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x1b000~17000 = 0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.799+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692f00 cost 148186 (13 ios * 4000 + 96186 bytes) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692f00 onodes 0x5557e9b95680 shared_blobs 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9b95788) update 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e9b95788) update inline shard 508 bytes from 26 extents 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:d313ddd1:::smithi13140796-49:head# is 927 (413 bytes onode + 2 bytes spanning blobs + 512 bytes inline extents) 2019-12-07T19:33:58.799+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692f00 allocated 0x[437000~17000] released 0x[] 2019-12-07T19:33:58.800+0000 7f2b78ce3700 10 freelist allocate 0x437000~17000 2019-12-07T19:33:58.800+0000 7f2b78ce3700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:33:58.800+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 prepare 2019-12-07T19:33:58.800+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7692f00 2019-12-07T19:33:58.800+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.800+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c4000 finish 2019-12-07T19:33:58.800+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.800+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 52 ==== MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763e800 con 0x5557e79cbc00 2019-12-07T19:33:58.800+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c54a0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.800+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e7954000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c54a0 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:154, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'7], crt=21'6) [] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (21'2) delete 1:c87708e9:::smithi13140796-17:head by client.4540.0:154 2019-12-07T19:33:58.792683+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e9b10900 seq 15 2019-12-07T19:33:58.800+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head#7 onode 0x5557e76caf00 txc 0x5557e9b10900 2019-12-07T19:33:58.800+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head#7 0x0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92310) put_ref 0x0~2000 Blob(0x5557e9d92310 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d922a0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7668690) put_ref 0x0~2000 Blob(0x5557e7668690 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36540 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74d9ea0) put_ref 0x0~2000 Blob(0x5557e74d9ea0 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d923f0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92070) put_ref 0x0~2000 Blob(0x5557e9d92070 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92460 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d924d0) put_ref 0x0~2000 Blob(0x5557e9d924d0 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92540 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d925b0) put_ref 0x0~2000 Blob(0x5557e9d925b0 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92620 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92690) put_ref 0x0~2000 Blob(0x5557e9d92690 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92700 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92770) put_ref 0x0~2000 Blob(0x5557e9d92770 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d927e0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92850) put_ref 0x0~1000 Blob(0x5557e9d92850 blob([0xd0000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9d928c0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e76cb008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e9d92310 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d922a0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xc0000~2000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92310) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d922b8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xc0000~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d922a0) put 0x5557e9d922a0 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d922b8 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e7668690 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d36540 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xc2000~2000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e7668690) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d36558 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xc2000~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d36540) put 0x5557e9d36540 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d36558 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x5557e74d9ea0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d923f0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xc4000~2000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74d9ea0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92408 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xc4000~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d923f0) put 0x5557e9d923f0 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92408 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x5557e9d92070 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d92460 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xc6000~2000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92070) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92478 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xc6000~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d92460) put 0x5557e9d92460 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92478 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x5557e9d924d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d92540 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xc8000~2000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d924d0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92558 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xc8000~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d92540) put 0x5557e9d92540 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92558 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x5557e9d925b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d92620 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xca000~2000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d925b0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92638 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xca000~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d92620) put 0x5557e9d92620 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92638 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x5557e9d92690 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d92700 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xcc000~2000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92690) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92718 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xcc000~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d92700) put 0x5557e9d92700 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92718 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x5557e9d92770 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d927e0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xce000~2000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92770) discard_unallocated 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d927f8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xce000~2000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d927e0) put 0x5557e9d927e0 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d927f8 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 aio_wait 2019-12-07T19:33:58.800+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692f00 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x10000~1000: 0x0~1000 Blob(0x5557e9d92850 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x5557e9d928c0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0xd0000~1000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92850) discard_unallocated 0x0~1000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d928d8 in 0x5557e62696c0) _discard 0x0~1000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0xd0000~1000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d928c0) put 0x5557e9d928c0 removing self from set 0x5557e75b40d0 2019-12-07T19:33:58.800+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 io_done 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d928d8 in 0x5557e62696c0) _clear 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.3s5_head 5#1:c87708e9:::smithi13140796-17:head#7 = 0 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.800+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10900 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10900 onodes shared_blobs 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10900 allocated 0x[] released 0x[c0000~11000] 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0xc0000~11000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 freelist release 0xc0000~11000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 prepare 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10900 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 io_done 2019-12-07T19:33:58.800+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c54a0 finish 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2067 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95680 had 1 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2090 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2090 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x5574d~82a to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x5574d 0x5574d~82a to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x2574d 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x74d 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xf77 and padding block with 0x89 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x55f77 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000122893s (0.000041713s flush + 0.000081180s kv commit) 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7692f00> 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 kv_submitted 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692f00 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1250 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692f00 finishing 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692f00 onodes 0x5557e9b95680 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9eb69b8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9e447e0 space 0x5557e9eb69b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080b08 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07ea20 space 0x5557ea080b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080c58 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07eab0 space 0x5557ea080c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080d38 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07eb40 space 0x5557ea080d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080e18 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07ebd0 space 0x5557ea080e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76caf00 had 1 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080ef8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07ec60 space 0x5557ea080ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea080fd8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07ecf0 space 0x5557ea080fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea0810b8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07ed80 space 0x5557ea0810b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea081198 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07ee10 space 0x5557ea081198 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea081278 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07eea0 space 0x5557ea081278 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea0813c8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07ef30 space 0x5557ea0813c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea0814a8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07efc0 space 0x5557ea0814a8 0x1000~1000 writing nocache) 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x55f77~4f9 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692f00 done 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692f00 [] 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x55f77 0x55f77~4f9 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x25f77 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xf77 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x470 and padding block with 0xb90 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.800+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=25, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cbc00 2019-12-07T19:33:58.800+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x56470 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.800+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000129775s (0.000000080s flush + 0.000129695s kv commit) 2019-12-07T19:33:58.800+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10900> 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 kv_submitted 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10900 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 finishing 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10900 onodes 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10900 done 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10900 [c0000~11000] 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0xc0000~11000 2019-12-07T19:33:58.800+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.800+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.800+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.800+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 21/18 ECSubWriteReply(tid=26, last_complete=21'7, committed=1, applied=1)) v2 -- 0x5557e9c76600 con 0x5557e79cbc00 2019-12-07T19:33:58.800+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.800+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 101 ==== MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e76ca500 con 0x5557e79cac00 2019-12-07T19:33:58.800+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c49a0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.800+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.800+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c49a0 prio 127 cost 0 latency 0.000061 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.801+0000 7f2b72cd7700 25 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.801+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~19000 size 0x19000 (102400) 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7b8f0 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7ba40 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bb20 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bb90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bc00 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bc70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bce0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bd50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bdc0 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7be30 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bea0 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bf10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bf80 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36000 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36070 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c360e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36150 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c361c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36230 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c362a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36310 blob([0x180000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9c36380 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b7b8f0 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b960 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b7ba40 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bab0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b7bb20 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bb90 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b7bc00 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bc70 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b7bce0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bd50 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b7bdc0 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7be30 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b7bea0 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bf10 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9b7bf80 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36000 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9c36070 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c360e0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9c36150 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c361c0 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9c36230 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c362a0 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9c36310 blob([0x180000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9c36380 sbid 0x0)) need {<0x0, 0x1000> : [0x18000:0~1000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~1000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c2b908 in 0x5557e6269880) _discard 0x1000~1000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9b598c0 space 0x5557e9c2b908 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b7b8f0 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7b978 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e44c60 space 0x5557e9b7b978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b7ba40 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bac8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e44bd0 space 0x5557e9b7bac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b7bb20 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bb90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bba8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e44cf0 space 0x5557e9b7bba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b7bc00 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bc70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bc88 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e44d80 space 0x5557e9b7bc88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b7bce0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bd50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bd68 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e44e10 space 0x5557e9b7bd68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b7bdc0 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7be30 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7be48 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e44ea0 space 0x5557e9b7be48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b7bea0 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bf10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9b7bf28 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e44f30 space 0x5557e9b7bf28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9b7bf80 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36000 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36018 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e44fc0 space 0x5557e9c36018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9c36070 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c360e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c360f8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557e9e45050 space 0x5557e9c360f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9c36150 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c361c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c361d8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557ea07f290 space 0x5557e9c361d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9c36230 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c362a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c362b8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557ea07f320 space 0x5557e9c362b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9c36310 blob([0x180000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9c36380 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18000:0~1000]} 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36398 in 0x5557e6269880) _discard 0x0~1000 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 _add level 1 near 0 on buffer(0x5557ea07f3b0 space 0x5557e9c36398 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 102400 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=102400 r=102400 len=102400 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 102400 2019-12-07T19:33:58.801+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read: Checking hash of 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.801+0000 7f2b72cd7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=81, attrs_read=0)) v2 -- 0x5557e9c7cf00 con 0x5557e79cac00 2019-12-07T19:33:58.801+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e75c49a0 finish 2019-12-07T19:33:58.801+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.801+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 53 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 96536+0+0 (crc 0 0 0) 0x5557e763d800 con 0x5557e79cbc00 2019-12-07T19:33:58.801+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c51e0 prio 127 cost 0 latency 0.000042 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.801+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c51e0 prio 127 cost 0 latency 0.000153 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.bs1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'7], crt=21'5) [21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'5 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'6 lcod 21'7 active mbc={}] rollforward: entry=21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'6 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'6 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.801+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b11b00 seq 16 2019-12-07T19:33:58.801+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:33:58.801+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.801+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x1c000~17000 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 1#1:dce63cc3:::smithi13140796-50:head# 0x1c000~17000 - have 0x1c000 (114688) bytes fadvise_flags 0x0 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.801+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x1c000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.801+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b11b00 12 blobs 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate 0x17000/1000,17000,0 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0xc0000~11000/1000,17000,0 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x44e000~6000/1000,17000,0 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0xc0000~11000,0x44e000~6000] 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea081b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea081ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea081b20 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081ce0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea081b20) get_ref 0x0~2000 Blob(0x5557ea081b20 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081ce0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea081b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557ea081b20 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea081ce0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557ea081cf8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d928c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92850 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d928c0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92850) get_ref 0x0~2000 Blob(0x5557e9d92850 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d928c0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9d92850 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d928c0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d928d8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea081f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea081f80 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92770 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea081f80) get_ref 0x0~2000 Blob(0x5557ea081f80 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92770 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea081f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557ea081f80 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92770 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92788 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d927e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d927e0 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92c40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d927e0) get_ref 0x0~2000 Blob(0x5557e9d927e0 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92c40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d927e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e9d927e0 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92c40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92c58 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea081f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea081f10 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92690 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea081f10) get_ref 0x0~2000 Blob(0x5557ea081f10 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92690 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea081f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557ea081f10 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92690 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d926a8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92700 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92bd0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92700) get_ref 0x0~2000 Blob(0x5557e9d92700 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92bd0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e9d92700 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92bd0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92be8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea081ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d925b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea081ea0 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d925b0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea081ea0) get_ref 0x0~2000 Blob(0x5557ea081ea0 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d925b0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea081ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x5557ea081ea0 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d925b0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d925c8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d924d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92b60 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d924d0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92b60) get_ref 0x0~2000 Blob(0x5557e9d92b60 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d924d0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x5557e9d92b60 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d924d0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d924e8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92540 blob([0xd0000~1000,0x44e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92af0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92540) get_ref 0x0~2000 Blob(0x5557e9d92540 blob([0xd0000~1000,0x44e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92af0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x5557e9d92540 blob([0xd0000~1000,0x44e000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92af0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92b08 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92070 blob([0x44f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92460 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92070) get_ref 0x0~2000 Blob(0x5557e9d92070 blob([0x44f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d92460 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x5557e9d92070 blob([0x44f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d92460 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92478 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d923f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92a80 blob([0x451000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d923f0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92a80) get_ref 0x0~2000 Blob(0x5557e9d92a80 blob([0x451000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d923f0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x5557e9d92a80 blob([0x451000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d923f0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d92408 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d929a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92a10 blob([0x453000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d929a0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92a10) get_ref 0x0~1000 Blob(0x5557e9d92a10 blob([0x453000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d929a0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d92a10) get_ref init 0x1000, 1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x32000~1000: 0x0~1000 Blob(0x5557e9d92a10 blob([0x453000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9d929a0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d929b8 in 0x5557e6269c00) _discard 0x0~1000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x33000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9eb4388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x1c000~17000 = 0 2019-12-07T19:33:58.802+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.802+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11b00 cost 152186 (14 ios * 4000 + 96186 bytes) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11b00 onodes 0x5557e9eb4280 shared_blobs 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9eb4388) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9eb4388) update inline shard 499 bytes from 26 extents 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 918 (413 bytes onode + 2 bytes spanning blobs + 503 bytes inline extents) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11b00 allocated 0x[c0000~11000,44e000~6000] released 0x[] 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 freelist allocate 0xc0000~11000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 freelist allocate 0x44e000~6000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 prepare 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b11b00 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'7 crt=21'6 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.802+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c51e0 finish 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.802+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 aio_wait 2019-12-07T19:33:58.802+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11b00 2019-12-07T19:33:58.802+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 io_done 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2087 < min_flush_size 524288 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4280 had 1 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2110 < min_flush_size 524288 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2110 < min_flush_size 524288 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x56470~83e to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x56470 0x56470~83e to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x26470 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x470 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xcae and padding block with 0x352 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x56cae 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.802+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000129959s (0.000038719s flush + 0.000091240s kv commit) 2019-12-07T19:33:58.802+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11b00> 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.802+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 kv_submitted 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11b00 2019-12-07T19:33:58.802+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 finishing 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11b00 onodes 0x5557e9eb4280 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92408 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07fdd0 space 0x5557e9d92408 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92478 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07fd40 space 0x5557e9d92478 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d924e8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07fc20 space 0x5557e9d924e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d925c8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07fb90 space 0x5557e9d925c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.802+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=27, last_complete=21'8, committed=1, applied=1)) v2 -- 0x5557e626f800 con 0x5557e79cbc00 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d926a8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07fa70 space 0x5557e9d926a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92788 in 0x5557e6269c00) _finish_write discard buffer(0x5557e79ab050 space 0x5557e9d92788 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d928d8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e9b59b00 space 0x5557e9d928d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d929b8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07fe60 space 0x5557e9d929b8 0x0~1000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92b08 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07fcb0 space 0x5557e9d92b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92be8 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07fb00 space 0x5557e9d92be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d92c58 in 0x5557e6269c00) _finish_write discard buffer(0x5557ea07e510 space 0x5557e9d92c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea081cf8 in 0x5557e6269c00) _finish_write discard buffer(0x5557e79ab170 space 0x5557ea081cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.802+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11b00 done 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.802+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11b00 [] 2019-12-07T19:33:58.802+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.802+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.802+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 102 ==== MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9ff8780 con 0x5557e79cac00 2019-12-07T19:33:58.802+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.802+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.802+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e7694b00 prio 127 cost 0 latency 0.000059 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.802+0000 7f2b76cdf700 25 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.802+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~19000 size 0x19000 (102400) 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7b8f0 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7ba40 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bb20 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bb90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bc00 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bc70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bce0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bd50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bdc0 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7be30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bea0 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bf10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bf80 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36070 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c360e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36150 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c361c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36230 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c362a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36310 blob([0x180000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9c36380 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.802+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 103 ==== MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e9c7cf00 con 0x5557e79cac00 2019-12-07T19:33:58.802+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 102400 2019-12-07T19:33:58.802+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d858c0 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.802+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.802+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=102400 r=102400 len=102400 2019-12-07T19:33:58.802+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.802+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.803+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.803+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 102400 2019-12-07T19:33:58.803+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:33:58.803+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:33:58.803+0000 7f2b76cdf700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read: Checking hash of 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.803+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=82, attrs_read=0)) v2 -- 0x5557e9bc0a00 con 0x5557e79cac00 2019-12-07T19:33:58.803+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e7694b00 finish 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.803+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d858c0 prio 127 cost 0 latency 0.000113 MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s1 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.803+0000 7f2b72cd7700 25 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.803+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~19000 size 0x19000 (102400) 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b707e0 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b8f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7b8f0 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7b960 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7ba40 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bb20 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bb90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bc00 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bc70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bce0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bd50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bdc0 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7be30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bea0 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7bf10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9b7bf80 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36070 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c360e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36150 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c361c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36230 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c362a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9c36310 blob([0x180000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9c36380 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 102400 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=102400 r=102400 len=102400 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 102400 2019-12-07T19:33:58.803+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.7s1_head 1#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read: Checking hash of 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.803+0000 7f2b72cd7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=83, attrs_read=0)) v2 -- 0x5557e76ca500 con 0x5557e79cac00 2019-12-07T19:33:58.803+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9d858c0 finish 2019-12-07T19:33:58.803+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.803+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 54 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e79e2800 con 0x5557e79cbc00 2019-12-07T19:33:58.803+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85ce0 prio 127 cost 0 latency 0.000041 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.803+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.803+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 55 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (crc 0 0 0) 0x5557e79e3000 con 0x5557e79cbc00 2019-12-07T19:33:58.803+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.803+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 prio 127 cost 0 latency 0.000375 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.bs1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log log((0'0,21'8], crt=21'6) [21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'6 lcod 21'8 active mbc={}] add_log_entry 21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'8 active mbc={}] rollforward: entry=21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'8 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.803+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b11800 seq 17 2019-12-07T19:33:58.803+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 3 keys 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 3 keys = 0 2019-12-07T19:33:58.803+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.803+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11800 onodes 0x5557e9eb4280 shared_blobs 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9eb4388) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 990 (485 bytes onode + 2 bytes spanning blobs + 503 bytes inline extents) 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 prepare 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11800 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 io_done 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'8 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 finish 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.803+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 127 cost 0 latency 0.000344 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.bs1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 lcod 21'8 active mbc={}] 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 lcod 21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.803+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 lcod 21'8 active mbc={}] append_log log((0'0,21'9], crt=21'7) [21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.803+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.803+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2030 < min_flush_size 524288 2019-12-07T19:33:58.803+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'9 crt=21'7 lcod 21'8 active mbc={}] add_log_entry 21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'9 crt=21'7 lcod 21'8 active mbc={}] append_log approx pg log length = 10 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb4280 had 1 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2053 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'9 crt=21'7 lcod 21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2053 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x56cae~805 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x56cae 0x56cae~805 to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x26cae 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xcae 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x4b3 and padding block with 0xb4d 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b10300 seq 18 2019-12-07T19:33:58.804+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 3 keys 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 3 keys = 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x574b3 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10300 onodes 0x5557e9b95680 shared_blobs 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95788) update 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000106475s (0.000000080s flush + 0.000106395s kv commit) 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11800> 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:d313ddd1:::smithi13140796-49:head# is 999 (485 bytes onode + 2 bytes spanning blobs + 512 bytes inline extents) 2019-12-07T19:33:58.804+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 kv_submitted 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11800 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.804+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 finishing 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11800 onodes 0x5557e9eb4280 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11800 done 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 prepare 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10300 prepare 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10300 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 io_done 2019-12-07T19:33:58.804+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11800 [] 2019-12-07T19:33:58.804+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'9 crt=21'7 lcod 21'8 active mbc={}] op_applied version 21'10 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2039 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95680 had 1 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2062 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2062 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x574b3~80e to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.804+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=28, last_complete=21'9, committed=1, applied=1)) v2 -- 0x5557e76b0e00 con 0x5557e79cbc00 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x574b3 0x574b3~80e to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x274b3 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x4b3 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xcc1 and padding block with 0x33f 2019-12-07T19:33:58.804+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x57cc1 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.804+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000128370s (0.000000064s flush + 0.000128306s kv commit) 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10300> 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.804+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 kv_submitted 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10300 2019-12-07T19:33:58.804+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 finishing 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10300 onodes 0x5557e9b95680 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10300 done 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.804+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10300 [] 2019-12-07T19:33:58.804+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.804+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=29, last_complete=21'10, committed=1, applied=1)) v2 -- 0x5557e7a9a200 con 0x5557e79cbc00 2019-12-07T19:33:58.804+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 56 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 33092+0+0 (crc 0 0 0) 0x5557e9fed000 con 0x5557e79cbc00 2019-12-07T19:33:58.804+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84dc0 prio 127 cost 0 latency 0.000029 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.804+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.bs1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'10], crt=21'7) [21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'10 crt=21'7 lcod 21'10 active mbc={}] add_log_entry 21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'10 crt=21'7 lcod 21'10 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'10 crt=21'7 lcod 21'10 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b10f00 seq 19 2019-12-07T19:33:58.804+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 22 keys 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# 22 keys = 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10f00 cost 36742 (1 ios * 4000 + 32742 bytes) 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10f00 onodes 0x5557e9b95400 shared_blobs 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95508) update 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head# is 30805 (30530 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10f00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 prepare 2019-12-07T19:33:58.804+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 io_done 2019-12-07T19:33:58.804+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.804+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'10 crt=21'7 lcod 21'10 active mbc={}] op_applied version 21'11 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 831 < min_flush_size 524288 2019-12-07T19:33:58.805+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 finish 2019-12-07T19:33:58.805+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 32742 < min_flush_size 524288 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95400 had 1 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 32765 < min_flush_size 524288 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 32765 < min_flush_size 524288 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x57cc1~7ffd to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x57cc1 0x57cc1~7ffd to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x27cc1 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xcc1 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xcbe and padding block with 0x342 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x5fcbe 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.805+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000168228s (0.000000052s flush + 0.000168176s kv commit) 2019-12-07T19:33:58.805+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.805+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.805+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10f00> 2019-12-07T19:33:58.805+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.805+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.805+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10f00 2019-12-07T19:33:58.805+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 finishing 2019-12-07T19:33:58.805+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10f00 onodes 0x5557e9b95400 2019-12-07T19:33:58.805+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 done 2019-12-07T19:33:58.805+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.805+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10f00 [] 2019-12-07T19:33:58.805+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.805+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.805+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.805+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=30, last_complete=21'11, committed=1, applied=1)) v2 -- 0x5557e9c76600 con 0x5557e79cbc00 2019-12-07T19:33:58.805+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.805+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 13 ==== osd_op(client.4540.0:159 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 225+0+491520 (crc 0 0 0) 0x5557e7a7ab00 con 0x5557e6f87800 2019-12-07T19:33:58.805+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85340 prio 63 cost 491520 latency 0.000016 epoch 21 osd_op(client.4540.0:159 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.805+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:159 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 491520 e21) 2019-12-07T19:33:58.805+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.805+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:159 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 491520 e21) queued 2019-12-07T19:33:58.805+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.805+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:159 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 491520 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.805+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85340 prio 63 cost 491520 latency 0.000058 osd_op(client.4540.0:159 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.805+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: osd_op(client.4540.0:159 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.805+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op: op osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.805+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.805+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.806+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'5 client.4540.0:5 dirty|data_digest s 32768 uv 5 dd 508e5253 alloc_hint [0 0 0]) 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 0x5557e9dcae00 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] ov 21'5 av 21'9 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op append 0~491520 in=491520b 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [write 32768~491520 in=491520b] 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op write 32768~491520 in=491520b 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] clean_regions modifiedclean_offsets: [0~32768,524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=0=[]:{} 2019-12-07T19:33:58.806+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 57 ==== MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557e76ca500 con 0x5557e79cbc00 2019-12-07T19:33:58.806+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x5557e9dcae00 op modify 2019-12-07T19:33:58.806+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] set mtime to 2019-12-07T19:33:58.805739+0000 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] final snapset 0=[]:{} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85600 prio 127 cost 0 latency 0.000052 MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [0~32768,524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx alloc reply 0x5557e9d59180 result 0 2019-12-07T19:33:58.806+0000 7f2b74cdb700 25 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.806+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~1000 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] calc_trim_to_aggressive limit = 21'8 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x33000 (208896) 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9cb1730 blob([0x41b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d10e00 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op order client.4540 tid 159 last was 5 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9cb1730 blob([0x41b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d10e00 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] new_repop rep_tid 15 on osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] new_repop: repgather(0x5557e9cc0000 0'0 rep_tid=15 committed?=0 r=0) 2019-12-07T19:33:58.806+0000 7f2b784e2700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] issue_repop rep_tid 15 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=21'9 tt=0'0 tid=15 reqid=client.4540.0:159 client_op=osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_hash_info: found on disk, size 8192 2019-12-07T19:33:58.806+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 524288 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'9 tt=0'0 tid=15 reqid=client.4540.0:159 client_op=osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[32768~491520]}) 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=21'9 tt=0'0 tid=15 reqid=client.4540.0:159 client_op=osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[32768~491520]}) 2019-12-07T19:33:58.806+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'9 tt=0'0 tid=15 reqid=client.4540.0:159 client_op=osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[32768~491520]}) 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(32768~491520:9) ) 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 32768 buffers are {32768~491520(491520)} 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): new_size start 32768 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): adding buffer_update 32768,491520 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9cb1730 blob([0x41b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d10e00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d10e18 in 0x5557e6269c00) _discard 0x0~1000 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 _add level 1 near 0 on buffer(0x5557e9cd63f0 space 0x5557e9d10e18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): to_append: {32768~491520(491520)} 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~1000 = 4096 2019-12-07T19:33:58.806+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): appending 32768~491520 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: Getting attr on 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: not in cache 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: found on disk, size 208896 2019-12-07T19:33:58.806+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# hinfo_key = 0 2019-12-07T19:33:58.806+0000 7f2b74cdb700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 21/18 ECSubReadReply(tid=31, attrs_read=0)) v2 -- 0x5557ea010000 con 0x5557e79cbc00 2019-12-07T19:33:58.806+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9d85600 finish 2019-12-07T19:33:58.806+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] encode_and_write: 1:33f5bf01:::smithi13140796-2:head new_size 524288 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 524288 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): marking append 32768 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(32768~491520:9) ) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:head={32768~491520(491520)}} 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=21'9 tt=0'0 tid=15 reqid=client.4540.0:159 client_op=osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[32768~491520]}) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:head=[32768~491520]} 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{32768~491520(491520)} 2019-12-07T19:33:58.807+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=15, reqid=client.4540.0:159, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9d83000 con 0x5557e79cac00 2019-12-07T19:33:58.807+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=15, reqid=client.4540.0:159, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9d81000 con 0x5557e7519800 2019-12-07T19:33:58.807+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=15, reqid=client.4540.0:159, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9ba0800 con 0x5557e79ca800 2019-12-07T19:33:58.807+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=15, reqid=client.4540.0:159, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9b9e800 con 0x5557e79cbc00 2019-12-07T19:33:58.807+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=15, reqid=client.4540.0:159, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e763c000 con 0x5557e79ca000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 earliest_dup_version = 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 trim 21'9 (21'5) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:159 2019-12-07T19:33:58.805739+0000 0 ObjectCleanRegions clean_offsets: [0~32768,524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log log((0'0,21'8], crt=21'8) [21'9 (21'5) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:159 2019-12-07T19:33:58.805739+0000 0 ObjectCleanRegions clean_offsets: [0~32768,524288~18446744073709027327], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] add_log_entry 21'9 (21'5) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:159 2019-12-07T19:33:58.805739+0000 0 ObjectCleanRegions clean_offsets: [0~32768,524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log approx pg log length = 9 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.807+0000 7f2b784e2700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9cc5500 seq 17 2019-12-07T19:33:58.807+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x2000~1e000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 0#1:33f5bf01:::smithi13140796-2:head# 0x2000~1e000 - have 0x2000 (8192) bytes fadvise_flags 0x0 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x2000~1e000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9cc5500 15 blobs 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1e000/1000,1e000,0 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x454000~1e000/1000,1e000,0 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x454000~1e000] 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b701c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b579d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b701c0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b579d0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b701c0) get_ref 0x0~2000 Blob(0x5557e9b701c0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b579d0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b701c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9b701c0 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b579d0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b579e8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b70230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b70230 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71b20 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b70230) get_ref 0x0~2000 Blob(0x5557e9b70230 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b71b20 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b70230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e9b70230 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b71b20 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9b71b38 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71ea0 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92000 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71ea0) get_ref 0x0~2000 Blob(0x5557e9b71ea0 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92000 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9b71ea0 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92000 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92018 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d11500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71f10 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d11500 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71f10) get_ref 0x0~2000 Blob(0x5557e9b71f10 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d11500 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557e9b71f10 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d11500 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9d11518 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b71f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b71f80 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92c40 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71f80) get_ref 0x0~2000 Blob(0x5557e9b71f80 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92c40 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9b71f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557e9b71f80 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92c40 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92c58 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f0bf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f0bf10 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92af0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9f0bf10) get_ref 0x0~2000 Blob(0x5557e9f0bf10 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92af0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9f0bf10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557e9f0bf10 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92af0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92b08 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e92a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e92a10 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92930 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92a10) get_ref 0x0~2000 Blob(0x5557e9e92a10 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92930 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92a10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9e92a10 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92930 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92948 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e92b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e92b60 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92a80 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92b60) get_ref 0x0~2000 Blob(0x5557e9e92b60 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92a80 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92b60) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9e92b60 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92a80 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92a98 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e929a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e928c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e929a0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e928c0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e929a0) get_ref 0x0~2000 Blob(0x5557e9e929a0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e928c0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e929a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557e9e929a0 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e928c0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e928d8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e92850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e927e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e92850 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e927e0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92850) get_ref 0x0~2000 Blob(0x5557e9e92850 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e927e0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x5557e9e92850 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e927e0 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e927f8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e92770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e92770 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92700 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92770) get_ref 0x0~2000 Blob(0x5557e9e92770 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92700 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x5557e9e92770 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92700 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92718 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e92690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e92690 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92620 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92690) get_ref 0x0~2000 Blob(0x5557e9e92690 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92620 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e92690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x5557e9e92690 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92620 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92638 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e925b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e925b0 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92540 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e925b0) get_ref 0x0~2000 Blob(0x5557e9e925b0 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92540 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e925b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x5557e9e925b0 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92540 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92558 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e924d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e924d0 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92460 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e924d0) get_ref 0x0~2000 Blob(0x5557e9e924d0 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92460 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e924d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x5557e9e924d0 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92460 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92478 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e923f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e923f0 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92380 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e923f0) get_ref 0x0~2000 Blob(0x5557e9e923f0 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92380 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.blob(0x5557e9e923f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x5557e9e923f0 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92380 sbid 0x0)) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.BufferSpace(0x5557e9e92398 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x20000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e76cb508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x2000~1e000 = 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5500 cost 248854 (31 ios * 4000 + 124854 bytes) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5500 onodes 0x5557e76cb400 shared_blobs 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e76cb508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e76cb508) update inline shard 307 bytes from 16 extents 2019-12-07T19:33:58.807+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 14 ==== osd_op(client.4540.0:160 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 225+0+376832 (crc 0 0 0) 0x5557e9c2c580 con 0x5557e6f87800 2019-12-07T19:33:58.807+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c46e0 prio 63 cost 376832 latency 0.000024 epoch 21 osd_op(client.4540.0:160 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 796 (483 bytes onode + 2 bytes spanning blobs + 311 bytes inline extents) 2019-12-07T19:33:58.807+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:160 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5500 allocated 0x[454000~1e000] released 0x[] 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 freelist allocate 0x454000~1e000 2019-12-07T19:33:58.807+0000 7f2b784e2700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 prepare 2019-12-07T19:33:58.807+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9cc5500 2019-12-07T19:33:58.807+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 15 ==== osd_op(client.4540.0:161 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 225+0+62 (crc 0 0 0) 0x5557e9c2c840 con 0x5557e6f87800 2019-12-07T19:33:58.807+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c49a0 prio 63 cost 62 latency 0.000008 epoch 21 osd_op(client.4540.0:161 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.807+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:161 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 62 e21) 2019-12-07T19:33:58.807+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 16 ==== osd_op(client.4540.0:162 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 225+0+0 (crc 0 0 0) 0x5557e9c2cb00 con 0x5557e6f87800 2019-12-07T19:33:58.807+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 63 cost 0 latency 0.000004 epoch 21 osd_op(client.4540.0:162 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.807+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:162 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.807+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.807+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:160 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) queued 2019-12-07T19:33:58.808+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op_applied version 21'9 2019-12-07T19:33:58.808+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] eval_repop repgather(0x5557e9cc0000 21'9 rep_tid=15 committed?=0 r=0) 2019-12-07T19:33:58.808+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d85340 finish 2019-12-07T19:33:58.808+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.808+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:161 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 62 e21) queued 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:160 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c46e0 prio 63 cost 376832 latency 0.000488 osd_op(client.4540.0:160 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: osd_op(client.4540.0:160 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op: op osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(write n=1 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'9 client.4540.0:159 dirty|data_digest s 524288 uv 9 dd 374b5d0a alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'9 client.4540.0:159 dirty|data_digest s 524288 uv 9 dd 374b5d0a alloc_hint [0 0 0]) 2019-12-07T19:33:58.808+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'9 client.4540.0:159 dirty|data_digest s 524288 uv 9 dd 374b5d0a alloc_hint [0 0 0]) 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=2 w=0)) 2019-12-07T19:33:58.808+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 aio_wait 2019-12-07T19:33:58.808+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5500 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 0x5557e9b13c00 2019-12-07T19:33:58.808+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 io_done 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] ov 21'9 av 21'10 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op append 0~376832 in=376832b 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [write 524288~376832 in=376832b] 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op write 524288~376832 in=376832b 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] clean_regions modifiedclean_offsets: [0~524288,901120~18446744073708650495], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=0=[]:{} 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x5557e9b13c00 op modify 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] set mtime to 2019-12-07T19:33:58.808039+0000 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] final snapset 0=[]:{} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [0~524288,901120~18446744073708650495], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx alloc reply 0x5557e7606500 result 0 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] calc_trim_to_aggressive limit = 21'8 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op order client.4540 tid 160 last was 159 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] new_repop rep_tid 16 on osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 834 < min_flush_size 524288 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] new_repop: repgather(0x5557e7967e60 0'0 rep_tid=16 committed?=0 r=0) 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1944 < min_flush_size 524288 2019-12-07T19:33:58.808+0000 7f2b744da700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] issue_repop rep_tid 16 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb400 had 1 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=21'10 tt=0'0 tid=16 reqid=client.4540.0:160 client_op=osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1967 < min_flush_size 524288 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1967 < min_flush_size 524288 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x5fcbe~7af to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 901120 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'10 tt=0'0 tid=16 reqid=client.4540.0:160 client_op=osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[524288~376832]}) 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=21'10 tt=0'0 tid=16 reqid=client.4540.0:160 client_op=osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[524288~376832]}) 2019-12-07T19:33:58.808+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'10 tt=0'0 tid=16 reqid=client.4540.0:160 client_op=osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[524288~376832]}) 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(32768~491520:9) Extent(524288~376832:10) ) 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 524288 buffers are {524288~376832(376832)} 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x5fcbe 0x5fcbe~7af to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.766439+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _allocate len 0x46d from 0 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _allocate len 0x46d from 1 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,acd190000 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd190000~10000/10000,10000,acd190000 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x70000 to include allocated 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 28 (was clean) 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x2fcbe 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xcbe 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x46d and padding block with 0xb93 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x70000 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): new_size start 524288 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): adding buffer_update 524288,376832 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): to_append: {524288~376832(376832)} 2019-12-07T19:33:58.808+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): appending 524288~376832 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x6046d 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (28) on file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]), flushing log 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 28 len 0x49 crc 0x62045c01) 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x1b000~1000 to file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x1b000 0x1b000~1000 to file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1b000 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x1c000 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:58.808+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000047 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 28 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000468547s (0.000043544s flush + 0.000425003s kv commit) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.808+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc5500> 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.808+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 kv_submitted 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5500 2019-12-07T19:33:58.808+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5500 finishing 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5500 onodes 0x5557e76cb400 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b579e8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e79aacf0 space 0x5557e9b579e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b71b38 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220000 space 0x5557e9b71b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d11518 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220120 space 0x5557e9d11518 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92018 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220090 space 0x5557e9e92018 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92398 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220750 space 0x5557e9e92398 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92478 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea2206c0 space 0x5557e9e92478 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92558 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220630 space 0x5557e9e92558 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92638 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea2205a0 space 0x5557e9e92638 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92718 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220510 space 0x5557e9e92718 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e927f8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220480 space 0x5557e9e927f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e928d8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea2203f0 space 0x5557e9e928d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92948 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea2202d0 space 0x5557e9e92948 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92a98 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220360 space 0x5557e9e92a98 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92b08 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220240 space 0x5557e9e92b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92c58 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea2201b0 space 0x5557e9e92c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5500 done 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.808+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5500 [] 2019-12-07T19:33:58.808+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.808+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.808+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 20 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e76b1e00 con 0x5557e79ca000 2019-12-07T19:33:58.808+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4580 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.808+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.808+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 104 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e76b0e00 con 0x5557e79cac00 2019-12-07T19:33:58.808+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c58c0 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.808+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.809+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 58 ==== MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (crc 0 0 0) 0x5557ea010000 con 0x5557e79cbc00 2019-12-07T19:33:58.809+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.809+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.809+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 59 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9ba1000 con 0x5557e79cbc00 2019-12-07T19:33:58.809+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.809+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c4420 prio 127 cost 0 latency 0.000065 MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpRead(1.bs1 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.809+0000 7f2b78ce3700 25 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.809+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~1000 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x32000 (204800) 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9d37ab0 blob([0x400000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d108c0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc blob Blob(0x5557e9d37ab0 blob([0x400000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d108c0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read waiting for aio 2019-12-07T19:33:58.809+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 36 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36000 con 0x5557e7519800 2019-12-07T19:33:58.809+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e7695b80 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.809+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _generate_read_result_bl blob Blob(0x5557e9d37ab0 blob([0x400000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d108c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d108d8 in 0x5557e6269c00) _discard 0x0~1000 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 _add level 1 near 0 on buffer(0x5557ea07e3f0 space 0x5557e9d108d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~1000 = 4096 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: found on disk, size 204800 2019-12-07T19:33:58.809+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:33:58.809+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 21/18 ECSubReadReply(tid=32, attrs_read=0)) v2 -- 0x5557e9c7db80 con 0x5557e79cbc00 2019-12-07T19:33:58.809+0000 7f2b78ce3700 10 osd.6 21 dequeue_op 0x5557e75c4420 finish 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.809+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 60 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b37a00 con 0x5557e79cbc00 2019-12-07T19:33:58.809+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5080 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.809+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c5e40 prio 127 cost 0 latency 0.000284 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 pg pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'11], crt=21'7) [] 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.809+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] encode_and_write: 1:33f5bf01:::smithi13140796-2:head new_size 901120 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.809+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b10c00 seq 20 2019-12-07T19:33:58.809+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10c00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10c00 onodes shared_blobs 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10c00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 prepare 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10c00 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 io_done 2019-12-07T19:33:58.809+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 901120 2019-12-07T19:33:58.809+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e75c5e40 finish 2019-12-07T19:33:58.809+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): marking append 524288 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.809+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(32768~491520:9) Extent(524288~376832:10) ) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:head={524288~376832(376832)}} 2019-12-07T19:33:58.809+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 71 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9ef3800 con 0x5557e79ca800 2019-12-07T19:33:58.809+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=21'10 tt=0'0 tid=16 reqid=client.4540.0:160 client_op=osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[524288~376832]}) 2019-12-07T19:33:58.809+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e76958c0 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.809+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:head=[524288~376832]} 2019-12-07T19:33:58.809+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{524288~376832(376832)} 2019-12-07T19:33:58.809+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=16, reqid=client.4540.0:160, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e763f000 con 0x5557e79cac00 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.809+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=16, reqid=client.4540.0:160, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e763c800 con 0x5557e7519800 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.809+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=16, reqid=client.4540.0:160, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9ba1800 con 0x5557e79ca800 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.809+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=16, reqid=client.4540.0:160, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e79e5000 con 0x5557e79cbc00 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x6046d~471 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x6046d 0x6046d~471 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3046d 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x46d 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x8de and padding block with 0x722 2019-12-07T19:33:58.809+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:160, at_version=21'10, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e79e3800 con 0x5557e79ca000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 earliest_dup_version = 0 2019-12-07T19:33:58.809+0000 7f2b744da700 20 trim 21'10 (21'9) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:160 2019-12-07T19:33:58.808039+0000 0 ObjectCleanRegions clean_offsets: [0~524288,901120~18446744073708650495], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x608de 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.809+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000095457s (0.000000070s flush + 0.000095387s kv commit) 2019-12-07T19:33:58.809+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log log((0'0,21'9], crt=21'8) [21'10 (21'9) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:160 2019-12-07T19:33:58.808039+0000 0 ObjectCleanRegions clean_offsets: [0~524288,901120~18446744073708650495], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.809+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.809+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10c00> 2019-12-07T19:33:58.809+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.809+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.809+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:33:58.809+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10c00 2019-12-07T19:33:58.809+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 finishing 2019-12-07T19:33:58.809+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10c00 onodes 2019-12-07T19:33:58.809+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 done 2019-12-07T19:33:58.809+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.809+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10c00 [] 2019-12-07T19:33:58.809+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.809+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.809+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.809+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'9 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] add_log_entry 21'10 (21'9) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:160 2019-12-07T19:33:58.808039+0000 0 ObjectCleanRegions clean_offsets: [0~524288,901120~18446744073708650495], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.809+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'9 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log approx pg log length = 10 2019-12-07T19:33:58.809+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'9 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.809+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.809+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=33, last_complete=21'11, committed=1, applied=1)) v2 -- 0x5557e7a9a200 con 0x5557e79cbc00 2019-12-07T19:33:58.809+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.809+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.809+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9b11200 seq 18 2019-12-07T19:33:58.809+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys 2019-12-07T19:33:58.809+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2 keys = 0 2019-12-07T19:33:58.809+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.809+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.809+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x20000~17000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 0#1:33f5bf01:::smithi13140796-2:head# 0x20000~17000 - have 0x20000 (131072) bytes fadvise_flags 0x0 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.809+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x20000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b11200 12 blobs 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.809+0000 7f2b744da700 10 fbmap_alloc 0x5557e6282e00 allocate 0x17000/1000,17000,0 2019-12-07T19:33:58.809+0000 7f2b744da700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x472000~17000/1000,17000,0 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x472000~17000] 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7620a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7620a80 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36770 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e7620a80) get_ref 0x0~2000 Blob(0x5557e7620a80 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d36770 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e7620a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x5557e7620a80 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d36770 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9d36788 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76218f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7621960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76218f0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621960 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e76218f0) get_ref 0x0~2000 Blob(0x5557e76218f0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7621960 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e76218f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x5557e76218f0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621960 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621978 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e933b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2230 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e933b0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9be2230) get_ref 0x0~2000 Blob(0x5557e9be2230 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e933b0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9be2230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x5557e9be2230 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e933b0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e933c8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2150 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93340 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9be2150) get_ref 0x0~2000 Blob(0x5557e9be2150 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93340 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9be2150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x5557e9be2150 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e93340 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e93358 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb33b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e932d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb33b0 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e932d0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9cb33b0) get_ref 0x0~2000 Blob(0x5557e9cb33b0 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e932d0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9cb33b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x5557e9cb33b0 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e932d0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e932e8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb3490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb3490 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93110 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9cb3490) get_ref 0x0~2000 Blob(0x5557e9cb3490 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93110 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9cb3490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x5557e9cb3490 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e93110 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e93128 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e93260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e930a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e93260 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e930a0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e93260) get_ref 0x0~2000 Blob(0x5557e9e93260 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e930a0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e93260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x5557e9e93260 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e930a0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e930b8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e931f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e931f0 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93030 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e931f0) get_ref 0x0~2000 Blob(0x5557e9e931f0 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93030 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e931f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x5557e9e931f0 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e93030 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e93048 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e93180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e93180 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92fc0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e93180) get_ref 0x0~2000 Blob(0x5557e9e93180 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e92fc0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e93180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x5557e9e93180 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e92fc0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e92fd8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e92f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e92f50 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93ab0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e92f50) get_ref 0x0~2000 Blob(0x5557e9e92f50 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93ab0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e92f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x5557e9e92f50 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e93ab0 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e93ac8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e93b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e93b20 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93b90 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e93b20) get_ref 0x0~2000 Blob(0x5557e9e93b20 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93b90 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e93b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x5557e9e93b20 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e93b90 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e93ba8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e93c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e93c00 blob([0x488000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93c70 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e93c00) get_ref 0x0~1000 Blob(0x5557e9e93c00 blob([0x488000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e93c70 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.blob(0x5557e9e93c00) get_ref init 0x1000, 1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x36000~1000: 0x0~1000 Blob(0x5557e9e93c00 blob([0x488000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9e93c70 sbid 0x0)) 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9e93c88 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x37000 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.809+0000 7f2b744da700 20 bluestore.extentmap(0x5557e76cb508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.809+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x20000~17000 = 0 2019-12-07T19:33:58.809+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.809+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.809+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.810+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.810+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11200 cost 192182 (24 ios * 4000 + 96182 bytes) 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11200 onodes 0x5557e76cb400 shared_blobs 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore.extentmap(0x5557e76cb508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore.extentmap(0x5557e76cb508) update inline shard 532 bytes from 28 extents 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 1021 (483 bytes onode + 2 bytes spanning blobs + 536 bytes inline extents) 2019-12-07T19:33:58.810+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11200 allocated 0x[472000~17000] released 0x[] 2019-12-07T19:33:58.810+0000 7f2b744da700 10 freelist allocate 0x472000~17000 2019-12-07T19:33:58.810+0000 7f2b744da700 20 freelist _xor first_key 0x400000 last_key 0x480000 2019-12-07T19:33:58.810+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 prepare 2019-12-07T19:33:58.810+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b11200 2019-12-07T19:33:58.810+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'9 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op_applied version 21'10 2019-12-07T19:33:58.810+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] eval_repop repgather(0x5557e7967e60 21'10 rep_tid=16 committed?=0 r=0) 2019-12-07T19:33:58.810+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c46e0 finish 2019-12-07T19:33:58.810+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:161 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 62 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c49a0 prio 63 cost 62 latency 0.002611 osd_op(client.4540.0:161 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: osd_op(client.4540.0:161 1.cs0 1.80fdafcc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op: op osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.810+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 aio_wait 2019-12-07T19:33:58.810+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11200 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.810+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 io_done 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(write n=2 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'10 client.4540.0:160 dirty|data_digest s 901120 uv 10 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'10 client.4540.0:160 dirty|data_digest s 901120 uv 10 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:33:58.810+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'10 client.4540.0:160 dirty|data_digest s 901120 uv 10 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(write n=3 w=0)) 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 0x5557e9dcca00 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2187 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] ov 21'10 av 21'11 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb400 had 1 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2210 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2210 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x608de~8a2 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] do_osd_op setxattr _header (55) in=62b 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x608de 0x608de~8a2 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x308de 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x8de 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x180 and padding block with 0xe80 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] make_writeable 1:33f5bf01:::smithi13140796-2:head done, snapset=0=[]:{} 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] finish_ctx 1:33f5bf01:::smithi13140796-2:head 0x5557e9dcca00 op modify 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] set mtime to 2019-12-07T19:33:58.808061+0000 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] final snapset 0=[]:{} in 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] finish_ctx object 1:33f5bf01:::smithi13140796-2:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] execute_ctx alloc reply 0x5557e9b94a00 result 0 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x61180 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] calc_trim_to_aggressive limit = 21'8 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op order client.4540 tid 161 last was 160 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000147455s (0.000045774s flush + 0.000101681s kv commit) 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11200> 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.810+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 kv_submitted 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11200 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] new_repop rep_tid 17 on osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.810+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11200 finishing 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11200 onodes 0x5557e76cb400 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7621978 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd7ef0 space 0x5557e7621978 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d36788 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9cd7e60 space 0x5557e9d36788 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e92fd8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220fc0 space 0x5557e9e92fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] new_repop: repgather(0x5557e9cc0240 0'0 rep_tid=17 committed?=0 r=0) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e93048 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220f30 space 0x5557e9e93048 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e930b8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220ea0 space 0x5557e9e930b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e93128 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220e10 space 0x5557e9e93128 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b784e2700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] issue_repop rep_tid 17 o 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e932e8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220870 space 0x5557e9e932e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e93358 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220900 space 0x5557e9e93358 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e933c8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea220990 space 0x5557e9e933c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e93ac8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea221050 space 0x5557e9e93ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e93ba8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557ea2210e0 space 0x5557e9e93ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e93c88 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9b59c20 space 0x5557e9e93c88 0x0~1000 writing nocache) 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11200 done 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.810+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11200 [] 2019-12-07T19:33:58.810+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] submit_transaction: op Op(1:33f5bf01:::smithi13140796-2:head v=21'11 tt=0'0 tid=17 reqid=client.4540.0:161 client_op=osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.810+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head projected size 901120 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] start_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'11 tt=0'0 tid=17 reqid=client.4540.0:161 client_op=osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_state_to_reads: Op(1:33f5bf01:::smithi13140796-2:head v=21'11 tt=0'0 tid=17 reqid=client.4540.0:161 client_op=osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'11 tt=0'0 tid=17 reqid=client.4540.0:161 client_op=osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(32768~491520:9) Extent(524288~376832:10) ) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] generate_transactions: 1:33f5bf01:::smithi13140796-2:head, current size is 901120 buffers are {} 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): new_size start 901120 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] operator(): 1:33f5bf01:::smithi13140796-2:head resetting hinfo to logical size 901120 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(32768~491520:9) Extent(524288~376832:10) ) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: written: {1:33f5bf01:::smithi13140796-2:head={}} 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=21'11 tt=0'0 tid=17 reqid=client.4540.0:161 client_op=osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: written_set: {1:33f5bf01:::smithi13140796-2:head=[]} 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] try_reads_to_commit: 1:33f5bf01:::smithi13140796-2:head,{} 2019-12-07T19:33:58.810+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=17, reqid=client.4540.0:161, at_version=21'11, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e79e4800 con 0x5557e79cac00 2019-12-07T19:33:58.810+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=17, reqid=client.4540.0:161, at_version=21'11, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e7a11800 con 0x5557e7519800 2019-12-07T19:33:58.810+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=17, reqid=client.4540.0:161, at_version=21'11, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9d83000 con 0x5557e79ca800 2019-12-07T19:33:58.810+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=17, reqid=client.4540.0:161, at_version=21'11, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9d82000 con 0x5557e79cbc00 2019-12-07T19:33:58.810+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 37 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36200 con 0x5557e7519800 2019-12-07T19:33:58.810+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e9d84f20 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.810+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:161, at_version=21'11, trim_to=0'0, roll_forward_to=21'8)) v2 -- 0x5557e9fed800 con 0x5557e79ca000 2019-12-07T19:33:58.810+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 earliest_dup_version = 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 trim 21'11 (21'10) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:161 2019-12-07T19:33:58.808061+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log log((0'0,21'10], crt=21'8) [21'11 (21'10) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:161 2019-12-07T19:33:58.808061+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'10 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] add_log_entry 21'11 (21'10) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:161 2019-12-07T19:33:58.808061+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'10 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log approx pg log length = 11 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'10 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.810+0000 7f2b784e2700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9d5ac00 seq 19 2019-12-07T19:33:58.810+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 3 keys 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 3 keys = 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5ac00 cost 5938 (1 ios * 4000 + 1938 bytes) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5ac00 onodes 0x5557e76cb400 shared_blobs 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore.extentmap(0x5557e76cb508) update 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:33f5bf01:::smithi13140796-2:head# is 1021 (483 bytes onode + 2 bytes spanning blobs + 536 bytes inline extents) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5ac00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 prepare 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5ac00 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 io_done 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 lua=21'10 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] op_applied version 21'11 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2117 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] eval_repop repgather(0x5557e9cc0240 21'11 rep_tid=17 committed?=0 r=0) 2019-12-07T19:33:58.810+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c49a0 finish 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb400 had 1 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2140 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2140 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.810+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x61180~85c to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.810+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x61180 0x61180~85c to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x31180 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x180 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.810+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x9dc and padding block with 0x624 2019-12-07T19:33:58.810+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.810+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4580 prio 196 cost 0 latency 0.002057 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.810+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.810+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4580 finish 2019-12-07T19:33:58.810+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.810+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.810+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c58c0 prio 196 cost 0 latency 0.002040 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.810+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c58c0 finish 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695b80 prio 196 cost 0 latency 0.001830 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695b80 finish 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e76958c0 prio 196 cost 0 latency 0.001570 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e76958c0 finish 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 21 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9bf2000 con 0x5557e79ca000 2019-12-07T19:33:58.811+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695600 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5080 prio 196 cost 0 latency 0.001748 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=15, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'9 tt=0'0 tid=15 reqid=client.4540.0:159 client_op=osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[32768~491520]}) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'8 crt=21'8 lcod 21'7 mlcod 21'7 active+clean] repop_all_committed: repop tid 15 all committed 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] eval_repop repgather(0x5557e9cc0000 21'9 rep_tid=15 committed?=1 r=0) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] commit: repgather(0x5557e9cc0000 21'9 rep_tid=15 committed?=1 r=0) 2019-12-07T19:33:58.811+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] log_op_stats osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 491520 outb 0 lat 0.005271 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] sending reply on osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x5557e9d59180 2019-12-07T19:33:58.811+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(159 smithi13140796-2 [append 0~491520] v21'9 uv9 ondisk = 0) v8 -- 0x5557e9d59180 con 0x5557e6f87800 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.811+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] publish_stats_to_osd 21:25 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] removing repgather(0x5557e9cc0000 21'9 rep_tid=15 committed?=1 r=0) 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] q front is repgather(0x5557e9cc0000 21'9 rep_tid=15 committed?=1 r=0) 2019-12-07T19:33:58.811+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 22 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9bf3000 con 0x5557e79ca000 2019-12-07T19:33:58.811+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.811+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 38 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9bf2c00 con 0x5557e7519800 2019-12-07T19:33:58.811+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9d84580 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] remove_repop repgather(0x5557e9cc0000 21'9 rep_tid=15 committed?=1 r=0) 2019-12-07T19:33:58.811+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e9d85ce0 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'9 tt=0'0 tid=15 reqid=client.4540.0:159 client_op=osd_op(client.4540.0:159 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~491520 in=491520b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[32768~491520]}) 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] try_finish_rmw: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(32768~491520:9) Extent(524288~376832:10) ) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c5080 finish 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x619dc 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.811+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.811+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.811+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.811+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000381571s (0.000000047s flush + 0.000381524s kv commit) 2019-12-07T19:33:58.811+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.811+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5ac00> 2019-12-07T19:33:58.811+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.811+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84f20 prio 196 cost 0 latency 0.000532 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 61 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c76c00 con 0x5557e79cbc00 2019-12-07T19:33:58.811+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 kv_submitted 2019-12-07T19:33:58.811+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5ac00 2019-12-07T19:33:58.811+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 finishing 2019-12-07T19:33:58.811+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5ac00 onodes 0x5557e76cb400 2019-12-07T19:33:58.811+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6000 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5ac00 done 2019-12-07T19:33:58.811+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.811+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5ac00 [] 2019-12-07T19:33:58.811+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84f20 finish 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695600 prio 196 cost 0 latency 0.000245 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695600 finish 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 62 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9bf2600 con 0x5557e79cbc00 2019-12-07T19:33:58.811+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6160 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 prio 196 cost 0 latency 0.000178 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d85ce0 finish 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6000 prio 196 cost 0 latency 0.000124 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6000 finish 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84580 prio 196 cost 0 latency 0.000237 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84580 finish 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:162 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6160 prio 196 cost 0 latency 0.000089 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6160 finish 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:162 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 63 cost 0 latency 0.003783 osd_op(client.4540.0:162 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: osd_op(client.4540.0:162 1.cs0 1.80fdafcc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] do_op: op osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] do_op osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(write n=2 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:33:58.811+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.811+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 105 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9bf2e00 con 0x5557e79cac00 2019-12-07T19:33:58.811+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf62c0 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 106 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9bf2a00 con 0x5557e79cac00 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6420 prio 196 cost 0 latency 0.000004 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf62c0 prio 196 cost 0 latency 0.000027 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf62c0 finish 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf6420 prio 196 cost 0 latency 0.000026 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 72 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9bf3200 con 0x5557e79ca800 2019-12-07T19:33:58.811+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6580 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf6420 finish 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.811+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 73 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e7a9a200 con 0x5557e79ca800 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.811+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf66e0 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6580 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.811+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=16, last_complete=21'10, committed=1, applied=1)) v2 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.811+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'10 tt=0'0 tid=16 reqid=client.4540.0:160 client_op=osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[524288~376832]}) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'9 crt=21'8 lcod 21'8 mlcod 21'8 active+clean] repop_all_committed: repop tid 16 all committed 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] eval_repop repgather(0x5557e7967e60 21'10 rep_tid=16 committed?=1 r=0) 2019-12-07T19:33:58.811+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] commit: repgather(0x5557e7967e60 21'10 rep_tid=16 committed?=1 r=0) 2019-12-07T19:33:58.811+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] log_op_stats osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 376832 outb 0 lat 0.004347 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] sending reply on osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x5557e7606500 2019-12-07T19:33:58.812+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(160 smithi13140796-2 [append 0~376832] v21'10 uv10 ondisk = 0) v8 -- 0x5557e7606500 con 0x5557e6f87800 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.812+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] publish_stats_to_osd 21:26 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] removing repgather(0x5557e7967e60 21'10 rep_tid=16 committed?=1 r=0) 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] q front is repgather(0x5557e7967e60 21'10 rep_tid=16 committed?=1 r=0) 2019-12-07T19:33:58.812+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] remove_repop repgather(0x5557e7967e60 21'10 rep_tid=16 committed?=1 r=0) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'10 tt=0'0 tid=16 reqid=client.4540.0:160 client_op=osd_op(client.4540.0:160 1.cs0 1:33f5bf01:::smithi13140796-2:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[524288~376832]}) 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] try_finish_rmw: ExtentCache( Extents(1:33f5bf01:::smithi13140796-2:head)[ Extent(524288~376832:10) ) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6580 finish 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf66e0 prio 196 cost 0 latency 0.000111 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=17, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:33f5bf01:::smithi13140796-2:head v=21'11 tt=0'0 tid=17 reqid=client.4540.0:161 client_op=osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'10 crt=21'8 lcod 21'9 mlcod 21'9 active+clean] repop_all_committed: repop tid 17 all committed 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] eval_repop repgather(0x5557e9cc0240 21'11 rep_tid=17 committed?=1 r=0) 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] commit: repgather(0x5557e9cc0240 21'11 rep_tid=17 committed?=1 r=0) 2019-12-07T19:33:58.812+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] log_op_stats osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 0 outb 0 lat 0.004420 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] sending reply on osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x5557e9b94a00 2019-12-07T19:33:58.812+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(161 smithi13140796-2 [setxattr (55)] v21'11 uv11 ondisk = 0) v8 -- 0x5557e9b94a00 con 0x5557e6f87800 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.812+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] publish_stats_to_osd 21:27 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] removing repgather(0x5557e9cc0240 21'11 rep_tid=17 committed?=1 r=0) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] q front is repgather(0x5557e9cc0240 21'11 rep_tid=17 committed?=1 r=0) 2019-12-07T19:33:58.812+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] remove_repop repgather(0x5557e9cc0240 21'11 rep_tid=17 committed?=1 r=0) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] requeue_op 0x5557e75c4160 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=21'11 tt=0'0 tid=17 reqid=client.4540.0:161 client_op=osd_op(client.4540.0:161 1.cs0 1:33f5bf01:::smithi13140796-2:head [setxattr _header (55) in=62b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'8 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:33f5bf01:::smithi13140796-2:head=[]}) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: starting commit on Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=18 reqid=client.4540.0:161 roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: op: Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=18 reqid=client.4540.0:161 roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.812+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=18, reqid=client.4540.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557e79e2800 con 0x5557e79cac00 2019-12-07T19:33:58.812+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=18, reqid=client.4540.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557e79e2000 con 0x5557e7519800 2019-12-07T19:33:58.812+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557e763c000 con 0x5557e79ca800 2019-12-07T19:33:58.812+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=18, reqid=client.4540.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557e763e800 con 0x5557e79cbc00 2019-12-07T19:33:58.812+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=18, reqid=client.4540.0:161, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557e763d800 con 0x5557e79ca000 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'8 lcod 21'10 mlcod 21'10 active+clean] append_log log((0'0,21'11], crt=21'8) [] 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] rollforward: entry=21'9 (21'5) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:159 2019-12-07T19:33:58.805739+0000 0 ObjectCleanRegions clean_offsets: [0~32768,524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] rollforward: entry=21'10 (21'9) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:160 2019-12-07T19:33:58.808039+0000 0 ObjectCleanRegions clean_offsets: [0~524288,901120~18446744073708650495], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] rollforward: entry=21'11 (21'10) modify 1:33f5bf01:::smithi13140796-2:head by client.4540.0:161 2019-12-07T19:33:58.808061+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] append_log approx pg log length = 11 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.812+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.812+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.812+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7a93800 seq 20 2019-12-07T19:33:58.812+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.812+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.812+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93800 onodes shared_blobs 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.812+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 prepare 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93800 2019-12-07T19:33:58.812+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 io_done 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf66e0 finish 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x619dc~471 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x619dc 0x619dc~471 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x319dc 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x9dc 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe4d and padding block with 0x1b3 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 63 cost 0 latency 0.004758 osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op: op osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_object_context: found obc in cache: 0x5557e760c240 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_object_context: 0x5557e760c240 1:33f5bf01:::smithi13140796-2:head rwstate(none n=0 w=0) oi: 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e7967560 snapset: 0=[]:{} 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x61e4d 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.812+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] find_object_context 1:33f5bf01:::smithi13140796-2:head @head oi=1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000122087s (0.000000074s flush + 0.000122013s kv commit) 2019-12-07T19:33:58.812+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.812+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.812+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op oi 1:33f5bf01:::smithi13140796-2:head(21'11 client.4540.0:161 dirty|data_digest s 901120 uv 11 dd 454b3612 alloc_hint [0 0 0]) 2019-12-07T19:33:58.812+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a93800> 2019-12-07T19:33:58.812+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.812+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 kv_submitted 2019-12-07T19:33:58.812+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93800 2019-12-07T19:33:58.812+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93800 finishing 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op obc obc(1:33f5bf01:::smithi13140796-2:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.812+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93800 onodes 2019-12-07T19:33:58.812+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93800 done 2019-12-07T19:33:58.812+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.812+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93800 [] 2019-12-07T19:33:58.812+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx 0x5557e9b13c00 2019-12-07T19:33:58.812+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~1] ov 21'11 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~1] 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_read 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] async_read noted for 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_all_avail_shards: checking acting 1(3) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_all_avail_shards: checking acting 2(4) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_all_avail_shards: checking acting 4(2) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_all_avail_shards: checking acting 5(1) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_all_avail_shards: checking acting 6(0) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] start_read_op: starting ReadOp(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.812+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 23 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9ef3400 con 0x5557e79ca000 2019-12-07T19:33:58.812+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6840 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_read_op: starting read ReadOp(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.812+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.cs3 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9ff9b80 con 0x5557e79cac00 2019-12-07T19:33:58.812+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.cs2 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9d59400 con 0x5557e79cbc00 2019-12-07T19:33:58.812+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.cs1 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9b94780 con 0x5557e79ca000 2019-12-07T19:33:58.812+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 39 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9ef2600 con 0x5557e7519800 2019-12-07T19:33:58.812+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e9d84420 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 -- 0x5557e9b94500 con 0x5557e62f5400 2019-12-07T19:33:58.812+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 63 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c77400 con 0x5557e79cbc00 2019-12-07T19:33:58.812+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x5557e9b94500 con 0x5557e62f5400 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_read_op: started ReadOp(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=1(3),4(2),5(1),6(0)) 2019-12-07T19:33:58.812+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf69a0 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.812+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e75c4c60 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.812+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf6840 prio 196 cost 0 latency 0.000114 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.812+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 74 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9ef2200 con 0x5557e79ca800 2019-12-07T19:33:58.812+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6b00 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf6840 finish 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84420 prio 196 cost 0 latency 0.000100 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9d84420 finish 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf69a0 prio 196 cost 0 latency 0.000110 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf69a0 finish 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6b00 prio 196 cost 0 latency 0.000105 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6b00 finish 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4c60 prio 127 cost 0 latency 0.000183 MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpRead(1.cs0 21/18 ECSubRead(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=0,4096,0}, subchunks={1:33f5bf01:::smithi13140796-2:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.812+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.812+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~1000 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~1000 size 0x37000 (225280) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read defaulting to buffered read 2019-12-07T19:33:58.812+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e7620700 blob([0x39000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7620690 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.812+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) read 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 0x0~1000 = 4096 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_hash_info: Getting attr on 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_hash_info: not in cache 1:33f5bf01:::smithi13140796-2:head 2019-12-07T19:33:58.812+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_hash_info: found on disk, size 225280 2019-12-07T19:33:58.812+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key 2019-12-07T19:33:58.812+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:33f5bf01:::smithi13140796-2:head# hinfo_key = 0 2019-12-07T19:33:58.812+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 107 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x5557e9b94780 con 0x5557e79cac00 2019-12-07T19:33:58.812+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 -- 0x5557e9d58c80 con 0x5557e62f5400 2019-12-07T19:33:58.812+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c4c60 finish 2019-12-07T19:33:58.812+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6c60 prio 127 cost 0 latency 0.000012 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.812+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b82cf7700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.6 v2:172.21.15.109:6818/36937 0 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x5557e9d58c80 con 0x5557e62f5400 2019-12-07T19:33:58.812+0000 7f2b82cf7700 15 osd.6 21 enqueue_op 0x5557e7695ce0 prio 127 cost 0 latency 0.000011 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b82cf7700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 108 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e76b1a00 con 0x5557e79cac00 2019-12-07T19:33:58.812+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.812+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6dc0 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.812+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6c60 prio 127 cost 0 latency 0.000034 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=19, attrs_read=0) 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.812+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.812+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=4(2),5(1),6(0)) 2019-12-07T19:33:58.812+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 64 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 ==== 4260+0+0 (crc 0 0 0) 0x5557e9ff9b80 con 0x5557e79cbc00 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf6c60 finish 2019-12-07T19:33:58.813+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf6f20 prio 127 cost 0 latency 0.000012 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:33:58.813+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.813+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 24 ==== MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 ==== 4260+0+0 (unknown 0 0 0) 0x5557e9d59400 con 0x5557e79ca000 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf6dc0 prio 196 cost 0 latency 0.000063 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.813+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf7080 prio 127 cost 0 latency 0.000008 epoch 21 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:33:58.813+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=18, last_complete=21'11, committed=1, applied=1)) v2 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_finish_rmw: Op(1:33f5bf01:::smithi13140796-2:head v=0'0 tt=0'0 tid=18 reqid=client.4540.0:161 roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf6dc0 finish 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695ce0 prio 127 cost 0 latency 0.000124 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=19, attrs_read=0) 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 6(0),4096]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=4(2),5(1)) 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695ce0 finish 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf6f20 prio 127 cost 0 latency 0.000129 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=19, attrs_read=0) 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 6(0),4096]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=5(1)) 2019-12-07T19:33:58.813+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf6f20 finish 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.813+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf7080 prio 127 cost 0 latency 0.000137 MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpReadReply(1.cs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=19, attrs_read=0) 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_read_reply Complete: ReadOp(tid=19, to_read={1:33f5bf01:::smithi13140796-2:head=read_request_t(to_read=[0,16384,0], need={1(3)=[0,8],4(2)=[0,8],5(1)=[0,8],6(0)=[0,8]}, want_attrs=0)}, complete={1:33f5bf01:::smithi13140796-2:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(3),4096, 4(2),4096, 5(1),4096, 6(0),4096]))}, priority=127, obj_to_source={1:33f5bf01:::smithi13140796-2:head=1(3),4(2),5(1),6(0)}, source_to_obj={1(3)=1:33f5bf01:::smithi13140796-2:head,4(2)=1:33f5bf01:::smithi13140796-2:head,5(1)=1:33f5bf01:::smithi13140796-2:head,6(0)=1:33f5bf01:::smithi13140796-2:head}, in_progress=) 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] objects_read_async_cb: got: {1:33f5bf01:::smithi13140796-2:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx 0x5557e9b13c00 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx 1:33f5bf01:::smithi13140796-2:head [read 0~1 out=1b] ov 21'11 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op 1:33f5bf01:::smithi13140796-2:head [read 0~1 out=1b] 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx alloc reply 0x5557e9d58c80 result 0 2019-12-07T19:33:58.813+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.813+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] log_op_stats osd_op(client.4540.0:162 1.cs0 1:33f5bf01:::smithi13140796-2:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 1 lat 0.005541 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.813+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] publish_stats_to_osd 21:28 2019-12-07T19:33:58.813+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(162 smithi13140796-2 [read 0~1 out=1b] v0'0 uv11 ondisk = 0) v8 -- 0x5557e9d58c80 con 0x5557e6f87800 2019-12-07T19:33:58.813+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf7080 finish 2019-12-07T19:33:58.813+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 109 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 1364+0+0 (crc 0 0 0) 0x5557e763d800 con 0x5557e79cac00 2019-12-07T19:33:58.819+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf71e0 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.819+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9bf71e0 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'8 (0'0,21'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'8], crt=21'8) [21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0] 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] add_log_entry 21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.819+0000 7f2b76cdf700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7630000 seq 20 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.7s1_head 0x5557e76a92c0) get_onode oid 1#1:e7f24557:::smithi13140796-34:head#9 key 0x818000000000000001e7f245'W!smithi13140796-34!='0xfffffffffffffffe0000000000000009'o' 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.7s1_head 0x5557e76a92c0) r -2 v.len 0 2019-12-07T19:33:58.819+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# -> 1#1:e7f24557:::smithi13140796-34:head#9 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head# -> 1#1:e7f24557:::smithi13140796-34:head#9 = 0 2019-12-07T19:33:58.819+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630000 cost 5014 (1 ios * 4000 + 1014 bytes) 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630000 onodes 0x5557e9b95180 shared_blobs 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore.extentmap(0x5557e9b95288) update 1#1:e7f24557:::smithi13140796-34:head#9 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:e7f24557:::smithi13140796-34:head#9 is 817 (485 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 prepare 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630000 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 io_done 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.819+0000 7f2b76cdf700 10 osd.6 21 dequeue_op 0x5557e9bf71e0 finish 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1509 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95180 had 1 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea010500 had 1 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x61e4d~5fc to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x61e4d 0x61e4d~5fc to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x31e4d 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe4d 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x449 and padding block with 0xbb7 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x62449 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.819+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000141468s (0.000000073s flush + 0.000141395s kv commit) 2019-12-07T19:33:58.819+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.819+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.819+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7630000> 2019-12-07T19:33:58.819+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.819+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 kv_submitted 2019-12-07T19:33:58.819+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630000 2019-12-07T19:33:58.819+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630000 finishing 2019-12-07T19:33:58.819+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630000 onodes 0x5557e9b95180 2019-12-07T19:33:58.819+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630000 done 2019-12-07T19:33:58.819+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.819+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630000 [] 2019-12-07T19:33:58.819+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.819+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.819+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=84, last_complete=21'9, committed=1, applied=1)) v2 -- 0x5557e9b36e00 con 0x5557e79cac00 2019-12-07T19:33:58.819+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 75 ==== MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 56941+0+0 (crc 0 0 0) 0x5557e763c000 con 0x5557e79ca800 2019-12-07T19:33:58.819+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf7340 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.819+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x5557e769f000 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e9bf7340 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (21'2) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:163 2019-12-07T19:33:58.819566+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (21'2) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:163 2019-12-07T19:33:58.819566+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.819+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e9d5a600 seq 9 2019-12-07T19:33:58.819+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 22 keys 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# 22 keys = 0 2019-12-07T19:33:58.819+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.1s2_head 2#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.819+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5a600 cost 60591 (1 ios * 4000 + 56591 bytes) 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5a600 onodes 0x5557e76cb680 shared_blobs 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore.extentmap(0x5557e76cb788) update 2#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 2#1:891f0d13:::smithi13140796-14:head# is 54532 (54390 bytes onode + 2 bytes spanning blobs + 140 bytes inline extents) 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5a600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a600 prepare 2019-12-07T19:33:58.819+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 110 ==== MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763e800 con 0x5557e79cac00 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5a600 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a600 io_done 2019-12-07T19:33:58.819+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf74a0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.819+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.819+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e9bf7340 finish 2019-12-07T19:33:58.819+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.819+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGOpItem(op=MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x5557e76bd000 2019-12-07T19:33:58.819+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9bf74a0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.819+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s1 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.819+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'9], crt=21'8) [] 2019-12-07T19:33:58.819+0000 7f2b72cd7700 20 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] rollforward: entry=21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.819+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.820+0000 7f2b72cd7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7630300 seq 21 2019-12-07T19:33:58.820+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head#9 onode 0x5557e9b95180 txc 0x5557e7630300 2019-12-07T19:33:58.820+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head#9 0x0 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9be2a80) put_ref 0x0~2000 Blob(0x5557e9be2a80 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5acb0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b70770) put_ref 0x0~2000 Blob(0x5557e9b70770 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36a80 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b9d0) put_ref 0x0~2000 Blob(0x5557e9b7b9d0 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36a10 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36af0) put_ref 0x0~2000 Blob(0x5557e9c36af0 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36b60 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36bd0) put_ref 0x0~2000 Blob(0x5557e9c36bd0 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36c40 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36cb0) put_ref 0x0~2000 Blob(0x5557e9c36cb0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36d20 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36d90) put_ref 0x0~2000 Blob(0x5557e9c36d90 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36e00 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36e70) put_ref 0x0~2000 Blob(0x5557e9c36e70 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36ee0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36f50) put_ref 0x0~2000 Blob(0x5557e9c36f50 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36fc0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b7a0) put_ref 0x0~2000 Blob(0x5557e9e5b7a0 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b810 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b880) put_ref 0x0~2000 Blob(0x5557e9e5b880 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b8f0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b960) put_ref 0x0~2000 Blob(0x5557e9e5b960 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b9d0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5ba40) put_ref 0x0~2000 Blob(0x5557e9e5ba40 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bab0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bb20) put_ref 0x0~2000 Blob(0x5557e9e5bb20 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bb90 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bc00) put_ref 0x0~2000 Blob(0x5557e9e5bc00 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bc70 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bce0) put_ref 0x0~2000 Blob(0x5557e9e5bce0 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bd50 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bdc0) put_ref 0x0~2000 Blob(0x5557e9e5bdc0 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5be30 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.extentmap(0x5557e9b95288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e9be2a80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5acb0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x235000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9be2a80) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5acc8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 _rm buffer(0x5557e9f006c0 space 0x5557e9e5acc8 0x0~1000 clean) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x235000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5acb0) put 0x5557e9e5acb0 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5acc8 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e9b70770 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c36a80 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x237000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b70770) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36a98 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x237000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9c36a80) put 0x5557e9c36a80 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36a98 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x5557e9b7b9d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c36a10 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x239000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9b7b9d0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36a28 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x239000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9c36a10) put 0x5557e9c36a10 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36a28 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x5557e9c36af0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c36b60 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x23b000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36af0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36b78 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x23b000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9c36b60) put 0x5557e9c36b60 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36b78 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x5557e9c36bd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c36c40 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x23d000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36bd0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36c58 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x23d000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9c36c40) put 0x5557e9c36c40 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36c58 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x5557e9c36cb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c36d20 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x23f000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36cb0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36d38 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x23f000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9c36d20) put 0x5557e9c36d20 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36d38 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x5557e9c36d90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c36e00 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x241000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36d90) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36e18 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x241000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9c36e00) put 0x5557e9c36e00 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36e18 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x5557e9c36e70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c36ee0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x243000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36e70) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36ef8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x243000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9c36ee0) put 0x5557e9c36ee0 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36ef8 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x5557e9c36f50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c36fc0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x245000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9c36f50) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36fd8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x245000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9c36fc0) put 0x5557e9c36fc0 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9c36fd8 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x5557e9e5b7a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5b810 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x247000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b7a0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b828 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x247000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5b810) put 0x5557e9e5b810 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b828 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x5557e9e5b880 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5b8f0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x249000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b880) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b908 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x249000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5b8f0) put 0x5557e9e5b8f0 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b908 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x5557e9e5b960 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5b9d0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x24b000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5b960) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b9e8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x24b000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5b9d0) put 0x5557e9e5b9d0 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5b9e8 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x5557e9e5ba40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5bab0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x24d000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5ba40) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bac8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x24d000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5bab0) put 0x5557e9e5bab0 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bac8 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x5557e9e5bb20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5bb90 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x24f000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bb20) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bba8 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x24f000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5bb90) put 0x5557e9e5bb90 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bba8 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x5557e9e5bc00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5bc70 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x251000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bc00) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bc88 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x251000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5bc70) put 0x5557e9e5bc70 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bc88 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x5557e9e5bce0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5bd50 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x253000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bce0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bd68 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x253000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5bd50) put 0x5557e9e5bd50 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5bd68 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x5557e9e5bdc0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5be30 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x255000~2000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.blob(0x5557e9e5bdc0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5be48 in 0x5557e6269880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x255000~2000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.sharedblob(0x5557e9e5be30) put 0x5557e9e5be30 removing self from set 0x5557e76a9390 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore.BufferSpace(0x5557e9e5be48 in 0x5557e6269880) _clear 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.7s1_head 1#1:e7f24557:::smithi13140796-34:head#9 = 0 2019-12-07T19:33:58.820+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7630300 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7630300 onodes shared_blobs 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7630300 allocated 0x[] released 0x[235000~22000] 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0x235000~22000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 freelist release 0x235000~22000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 prepare 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7630300 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 io_done 2019-12-07T19:33:58.820+0000 7f2b72cd7700 10 osd.6 21 dequeue_op 0x5557e9bf74a0 finish 2019-12-07T19:33:58.820+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.820+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 65 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 ==== 1364+0+0 (crc 0 0 0) 0x5557e79e2800 con 0x5557e79cbc00 2019-12-07T19:33:58.820+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf7600 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.820+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9bf7600 prio 127 cost 0 latency 0.000104 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 pg pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'11 (0'0,21'11] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log log((0'0,21'11], crt=21'11) [21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0] 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] add_log_entry 21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.820+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b11500 seq 21 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.bs1_head 0x5557e75b41e0) get_onode oid 1#1:d07269f7:::smithi13140796-28:head#c key 0x818000000000000001d07269f721736d'ithi13140796-28!='0xfffffffffffffffe000000000000000c'o' 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.bs1_head 0x5557e75b41e0) r -2 v.len 0 2019-12-07T19:33:58.820+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# -> 1#1:d07269f7:::smithi13140796-28:head#c 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head# -> 1#1:d07269f7:::smithi13140796-28:head#c = 0 2019-12-07T19:33:58.820+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11500 cost 5014 (1 ios * 4000 + 1014 bytes) 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11500 onodes 0x5557e9b95400 shared_blobs 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95508) update 1#1:d07269f7:::smithi13140796-28:head#c 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:d07269f7:::smithi13140796-28:head#c is 30805 (30530 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 prepare 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11500 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 io_done 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] op_applied version 21'12 2019-12-07T19:33:58.820+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9bf7600 finish 2019-12-07T19:33:58.820+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 23479 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 56247 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 56469 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cb680 had 1 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 57719 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95180 had 1 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 89015 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 89228 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95400 had 1 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95b80 had 1 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 89251 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 89251 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x62449~15ca3 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x62449 0x62449~15ca3 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.809724+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _allocate len 0x80ec from 0 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _allocate len 0x80ec from 1 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,acd1a0000 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd1a0000~10000/10000,10000,acd1a0000 2019-12-07T19:33:58.820+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x80000 to include allocated 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 29 (was clean) 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x32449 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x449 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xec and padding block with 0xf14 2019-12-07T19:33:58.820+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x80000 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x780ec 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (29) on file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]), flushing log 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 29 len 0x49 crc 0x4d80c1c) 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x1c000~1000 to file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x1c000 0x1c000~1000 to file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1c000 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x1d000 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:33:58.821+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000020 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 29 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 3 cleaned 0 in 0.001000470s (0.000000077s flush + 0.001000393s kv commit) 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.821+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5a600,0x5557e7630300,0x5557e9b11500> 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a600 kv_submitted 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5a600 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a600 finishing 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5a600 onodes 0x5557e76cb680 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a600 done 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5a600 [] 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 kv_submitted 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7630300 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7630300 finishing 2019-12-07T19:33:58.821+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7630300 onodes 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7630300 done 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7630300 [235000~22000] 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0x235000~22000 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 kv_submitted 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11500 2019-12-07T19:33:58.821+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=45, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e765e800 con 0x5557e79ca800 2019-12-07T19:33:58.821+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11500 finishing 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11500 onodes 0x5557e9b95400 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11500 done 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11500 [] 2019-12-07T19:33:58.821+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.821+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.821+0000 7f2b76cdf700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=85, last_complete=21'9, committed=1, applied=1)) v2 -- 0x5557e9c76600 con 0x5557e79cac00 2019-12-07T19:33:58.821+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=34, last_complete=21'12, committed=1, applied=1)) v2 -- 0x5557e9b36000 con 0x5557e79cbc00 2019-12-07T19:33:58.821+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 66 ==== MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d82000 con 0x5557e79cbc00 2019-12-07T19:33:58.822+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf7760 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.822+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) pg 0x5557e7957000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9bf7760 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 pg pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs1 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log log((0'0,21'12], crt=21'11) [] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] rollforward: entry=21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.822+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 17 ==== osd_op(client.4540.0:166 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 225+0+55 (crc 0 0 0) 0x5557e7a7b080 con 0x5557e6f87800 2019-12-07T19:33:58.822+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf7a20 prio 63 cost 55 latency 0.000013 epoch 21 osd_op(client.4540.0:166 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:33:58.822+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:166 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 55 e21) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e9b10900 seq 22 2019-12-07T19:33:58.822+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head#c onode 0x5557e9b95400 txc 0x5557e9b10900 2019-12-07T19:33:58.822+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head#c 0x0 2019-12-07T19:33:58.822+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b70690) put_ref 0x0~2000 Blob(0x5557e9b70690 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2ba40 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4540 172.21.15.131:0/224485236 18 ==== osd_op(client.4540.0:167 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 225+0+0 (crc 0 0 0) 0x5557e7a7b340 con 0x5557e6f87800 2019-12-07T19:33:58.822+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:166 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 55 e21) queued 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d93180) put_ref 0x0~2000 Blob(0x5557e9d93180 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93260 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf7ce0 prio 63 cost 0 latency 0.000008 epoch 21 osd_op(client.4540.0:167 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.822+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74d9730) put_ref 0x0~2000 Blob(0x5557e74d9730 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9be2b60 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.822+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:166 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 55 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d932d0) put_ref 0x0~2000 Blob(0x5557e9d932d0 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93340 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d933b0) put_ref 0x0~2000 Blob(0x5557e9d933b0 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93420 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b571f0) put_ref 0x0~2000 Blob(0x5557e9b571f0 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb05b0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0620) put_ref 0x0~2000 Blob(0x5557e9cb0620 blob([0x1a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0690 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0700) put_ref 0x0~2000 Blob(0x5557e9cb0700 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0770 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb07e0) put_ref 0x0~2000 Blob(0x5557e9cb07e0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0850 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf7a20 prio 63 cost 55 latency 0.000045 osd_op(client.4540.0:166 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb08c0) put_ref 0x0~2000 Blob(0x5557e9cb08c0 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0930 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb09a0) put_ref 0x0~2000 Blob(0x5557e9cb09a0 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0a10 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0a80) put_ref 0x0~2000 Blob(0x5557e9cb0a80 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0af0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0b60) put_ref 0x0~2000 Blob(0x5557e9cb0b60 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0bd0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: osd_op(client.4540.0:166 1.cs0 1.b42e5efc (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0c40) put_ref 0x0~2000 Blob(0x5557e9cb0c40 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e9b95508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e9b70690 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c2ba40 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x19c000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b70690) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9c2ba58 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 _rm buffer(0x5557e9e450e0 space 0x5557e9c2ba58 0x0~1000 clean) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x19c000~2000 2019-12-07T19:33:58.822+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op: op osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9c2ba40) put 0x5557e9c2ba40 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9c2ba58 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e9d93180 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d93260 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x19e000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d93180) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93278 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x19e000~2000 2019-12-07T19:33:58.822+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d93260) put 0x5557e9d93260 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93278 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x5557e74d9730 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9be2b60 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1a0000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e74d9730) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9be2b78 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1a0000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9be2b60) put 0x5557e9be2b60 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9be2b78 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x5557e9d932d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d93340 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1a2000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d932d0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93358 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1a2000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d93340) put 0x5557e9d93340 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93358 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x5557e9d933b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d93420 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1a4000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d933b0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93438 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1a4000~2000 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_object_context: found obc in cache: 0x5557e9b21680 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9d93420) put 0x5557e9d93420 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93438 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x5557e9b571f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb05b0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1a6000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b571f0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb05c8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1a6000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb05b0) put 0x5557e9cb05b0 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb05c8 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x5557e9cb0620 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb0690 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1a8000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0620) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb06a8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(none n=0 w=0) oi: 1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1a8000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb0690) put 0x5557e9cb0690 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb06a8 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x5557e9cb0700 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb0770 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1aa000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0700) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0788 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1aa000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb0770) put 0x5557e9cb0770 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0788 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x5557e9cb07e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb0850 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1ac000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb07e0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0868 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1ac000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb0850) put 0x5557e9cb0850 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0868 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x5557e9cb08c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb0930 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b744da700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(21'6 client.4540.0:26 dirty|data_digest s 147456 uv 6 dd 2d9cfd61 alloc_hint [0 0 0]) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1ae000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb08c0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0948 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1ae000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb0930) put 0x5557e9cb0930 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0948 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x5557e9cb09a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb0a10 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1b0000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb09a0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0a28 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op obc obc(1:3f7a742d:::smithi13140796-9:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1b0000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb0a10) put 0x5557e9cb0a10 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0a28 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x5557e9cb0a80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb0af0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1b2000~2000] 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx 0x5557e9dcca00 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0a80) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0b08 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1b2000~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb0af0) put 0x5557e9cb0af0 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0b08 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x5557e9cb0b60 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb0bd0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1b4000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0b60) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0be8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1b4000~2000 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] ov 21'6 av 21'12 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb0bd0) put 0x5557e9cb0bd0 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0be8 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x5557e9cb0c40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x1b6000~2000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0c40) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0cc8 in 0x5557e6269c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x1b6000~2000 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.sharedblob(0x5557e9cb0cb0) put 0x5557e9cb0cb0 removing self from set 0x5557e75b42b0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0cc8 in 0x5557e6269c00) _clear 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op copy-from ver 6 in=55b 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.bs1_head 1#1:d07269f7:::smithi13140796-28:head#c = 0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:33:58.822+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] start_copy 1:3f7a742d:::smithi13140796-9:head from 1:9da56077:::smithi13140796-13:head @1 v6 flags 0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.822+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10900 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10900 onodes shared_blobs 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10900 allocated 0x[] released 0x[19c000~1c000] 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0x19c000~1c000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 freelist release 0x19c000~1c000 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _copy_some obc(1:3f7a742d:::smithi13140796-9:head rwstate(write n=1 w=0)) 0x5557e9bf4010 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 prepare 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10900 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 io_done 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.822+0000 7f2b74cdb700 10 osd.6 21 dequeue_op 0x5557e9bf7760 finish 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1250 < min_flush_size 524288 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b95400 had 1 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:33:58.822+0000 7f2b744da700 1 --2- >> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] conn(0x5557e9bf4400 0x5557e75b3180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x780ec~4f9 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x780ec 0x780ec~4f9 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x480ec 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xec 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x5e5 and padding block with 0xa1b 2019-12-07T19:33:58.822+0000 7f2b744da700 1 -- --> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] -- osd_op(unknown.0.13:1 1.9s0 1:9da56077:::smithi13140796-13:head [assert-version v6,copy-get max 8388608 in=29b] snapc 0=[] ondisk+read+known_if_redirected e21) v8 -- 0x5557e9c2c840 con 0x5557e9bf4400 2019-12-07T19:33:58.822+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 76 ==== MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9d83000 con 0x5557e79ca800 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:33:58.822+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9bf7a20 finish 2019-12-07T19:33:58.822+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e9bf7e40 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.822+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.822+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf7ce0 prio 63 cost 0 latency 0.000284 osd_op(client.4540.0:167 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGOpItem(op=MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x5557e769f000 2019-12-07T19:33:58.822+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: osd_op(client.4540.0:167 1.cs0 1.b42e5efc (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e9bf7e40 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s2 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.822+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op: op osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x785e5 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (21'2) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:163 2019-12-07T19:33:58.819566+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000151363s (0.000000079s flush + 0.000151284s kv commit) 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.822+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10900> 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 kv_submitted 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10900 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10900 finishing 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10900 onodes 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10900 done 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10900 [19c000~1c000] 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0x19c000~1c000 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.822+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.822+0000 7f2b75cdd700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.822+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] wait_for_blocked_object 1:3f7a742d:::smithi13140796-9:head 0x5557e9bf7ce0 2019-12-07T19:33:58.822+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf7ce0 finish 2019-12-07T19:33:58.822+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e9d5a000 seq 10 2019-12-07T19:33:58.822+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=35, last_complete=21'12, committed=1, applied=1)) v2 -- 0x5557e7a9a600 con 0x5557e79cbc00 2019-12-07T19:33:58.822+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.822+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5a000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5a000 onodes shared_blobs 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5a000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a000 prepare 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5a000 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a000 io_done 2019-12-07T19:33:58.822+0000 7f2b75cdd700 10 osd.6 21 dequeue_op 0x5557e9bf7e40 finish 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x785e5~471 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x785e5 0x785e5~471 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.822+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x485e5 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x5e5 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa56 and padding block with 0x5aa 2019-12-07T19:33:58.822+0000 7f2b957b2700 1 --2- >> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] conn(0x5557e9bf4400 0x5557e75b3180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.822+0000 7f2b957b2700 1 --2- >> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] conn(0x5557e9bf4400 0x5557e75b3180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.131:6819/37078 says I am v2:172.21.15.109:54618/0 (socket says 172.21.15.109:54618) 2019-12-07T19:33:58.822+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 learned_addr learned my addr 172.21.15.109:0/36937 (peer_addr_for_me v2:172.21.15.109:0/0) 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x78a56 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.822+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000125021s (0.000000049s flush + 0.000124972s kv commit) 2019-12-07T19:33:58.822+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5a000> 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a000 kv_submitted 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5a000 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a000 finishing 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5a000 onodes 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a000 done 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5a000 [] 2019-12-07T19:33:58.822+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.822+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.822+0000 7f2b79ce5700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=46, last_complete=21'3, committed=1, applied=1)) v2 -- 0x5557e9b36e00 con 0x5557e79ca800 2019-12-07T19:33:58.822+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.823+0000 7f2b957b2700 1 --2- 172.21.15.109:0/36937 >> [v2:172.21.15.131:6819/37078,v1:172.21.15.131:6821/37078] conn(0x5557e9bf4400 0x5557e75b3180 crc :-1 s=READY pgs=17 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:58.824+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 111 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=90, reqid=client.4540.0:174, at_version=21'13, trim_to=0'0, roll_forward_to=21'12)) v2 ==== 1364+0+0 (crc 0 0 0) 0x5557e9fed800 con 0x5557e79cac00 2019-12-07T19:33:58.824+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e76954a0 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=90, reqid=client.4540.0:174, at_version=21'13, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.824+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=90, reqid=client.4540.0:174, at_version=21'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.824+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.824+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=90, reqid=client.4540.0:174, at_version=21'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.824+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.824+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=90, reqid=client.4540.0:174, at_version=21'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.824+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e76954a0 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=90, reqid=client.4540.0:174, at_version=21'13, trim_to=0'0, roll_forward_to=21'12)) v2 pg pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] 2019-12-07T19:33:58.824+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=90, reqid=client.4540.0:174, at_version=21'13, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.824+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'12 (0'0,21'12] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log log((0'0,21'12], crt=21'12) [21'13 (21'11) delete 1:14fc029d:::smithi13140796-41:head by client.4540.0:174 2019-12-07T19:33:58.823507+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 0, new_object: 0] 2019-12-07T19:33:58.824+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={}] add_log_entry 21'13 (21'11) delete 1:14fc029d:::smithi13140796-41:head by client.4540.0:174 2019-12-07T19:33:58.823507+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.824+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={}] append_log approx pg log length = 13 2019-12-07T19:33:58.824+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.824+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.824+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.824+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9b10300 seq 24 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:14fc029d:::smithi13140796-41:head#d key 0x85800000000000000114fc029d21736d'ithi13140796-41!='0xfffffffffffffffe000000000000000d'o' 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) r -2 v.len 0 2019-12-07T19:33:58.824+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# -> 5#1:14fc029d:::smithi13140796-41:head#d 2019-12-07T19:33:58.824+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# -> 5#1:14fc029d:::smithi13140796-41:head#d = 0 2019-12-07T19:33:58.824+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000021.00000000000000000013' <- 0000000021.00000000000000000013 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.824+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.824+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10300 cost 5014 (1 ios * 4000 + 1014 bytes) 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10300 onodes 0x5557ea011180 shared_blobs 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011288) update 5#1:14fc029d:::smithi13140796-41:head#d 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:14fc029d:::smithi13140796-41:head#d is 893 (485 bytes onode + 2 bytes spanning blobs + 406 bytes inline extents) 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.824+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 prepare 2019-12-07T19:33:58.824+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10300 2019-12-07T19:33:58.824+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 io_done 2019-12-07T19:33:58.824+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.824+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.824+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={}] op_applied version 21'13 2019-12-07T19:33:58.824+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1585 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e76954a0 finish 2019-12-07T19:33:58.824+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.824+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea011180 had 1 2019-12-07T19:33:58.824+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.824+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9b7cf00 had 1 2019-12-07T19:33:58.824+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1608 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1608 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.824+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x78a56~648 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.824+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x78a56 0x78a56~648 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x48a56 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa56 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x9e and padding block with 0xf62 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x7909e 2019-12-07T19:33:58.825+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.825+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.825+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000137720s (0.000000043s flush + 0.000137677s kv commit) 2019-12-07T19:33:58.825+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.825+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.825+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10300> 2019-12-07T19:33:58.825+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.825+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 kv_submitted 2019-12-07T19:33:58.825+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10300 2019-12-07T19:33:58.825+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10300 finishing 2019-12-07T19:33:58.825+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10300 onodes 0x5557ea011180 2019-12-07T19:33:58.825+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10300 done 2019-12-07T19:33:58.825+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.825+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10300 [] 2019-12-07T19:33:58.825+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.825+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.825+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.825+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=90, last_complete=21'13, committed=1, applied=1)) v2 -- 0x5557e76b1c00 con 0x5557e79cac00 2019-12-07T19:33:58.825+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.827+0000 7f2b957b2700 1 -- 172.21.15.109:0/36937 <== osd.1 v2:172.21.15.131:6819/37078 1 ==== osd_op_reply(1 smithi13140796-13 [assert-version v6,copy-get max 8388608 out=213240b] v0'0 uv6 ondisk = 212992) v8 ==== 203+0+213240 (crc 0 0 0) 0x5557e9d58c80 con 0x5557e9bf4400 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] process_copy_chunk 1:3f7a742d:::smithi13140796-9:head tid 1 (0) Success 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] process_copy_chunk got digest: rx data 0x578a13d9 omap 0xffffffff, source: data 0x578a13d9 omap 0xffffffff flags 1 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] process_copy_chunk success; committing 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] process_copy_chunk complete r = (0) Success 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx 0x5557e9dcca00 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] ov 21'6 av 21'12 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_osd_op copy-from ver 6 in=55b 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] finish_copyfrom on 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] finish_copyfrom: exists, removing 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] fill_in_final_tx: writing directly to final object 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _write_copy_chunk 0x5557e9bf4010 0 attrs 212992 bytes 0 omap header bytes 0 omap data bytes 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] finish_copyfrom clearing omap flag on 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] make_writeable 1:3f7a742d:::smithi13140796-9:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] make_writeable 1:3f7a742d:::smithi13140796-9:head done, snapset=0=[]:{} 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] finish_ctx 1:3f7a742d:::smithi13140796-9:head 0x5557e9dcca00 op modify 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] mtime unchanged at 2019-12-07T19:33:58.638318+0000 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] final snapset 0=[]:{} in 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] finish_ctx object 1:3f7a742d:::smithi13140796-9:head marks clean_regions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] finish_ctx extra_reqids [client.4540.0:38,6,client.4540.0:37,4] {} 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] execute_ctx alloc reply 0x5557ea011b80 result 0 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] calc_trim_to_aggressive limit = 21'11 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] op order client.4540 tid 166 last was 26 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] new_repop rep_tid 20 on osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] new_repop: repgather(0x5557e9cc05a0 0'0 rep_tid=20 committed?=0 r=0) 2019-12-07T19:33:58.827+0000 7f2b7cceb700 7 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] issue_repop rep_tid 20 o 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] submit_transaction: op Op(1:3f7a742d:::smithi13140796-9:head v=21'12 tt=0'0 tid=20 reqid=client.4540.0:166 client_op=osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_hash_info: Getting attr on 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_hash_info: not in cache 1:3f7a742d:::smithi13140796-9:head 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) stat 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_hash_info: found on disk, size 36864 2019-12-07T19:33:58.827+0000 7f2b7cceb700 15 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) getattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key = 0 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): delete, setting projected size to 0 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): 1:3f7a742d:::smithi13140796-9:head projected size 212992 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] start_rmw: Op(1:3f7a742d:::smithi13140796-9:head v=21'12 tt=0'0 tid=20 reqid=client.4540.0:166 client_op=osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~212992]}) 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_state_to_reads: Op(1:3f7a742d:::smithi13140796-9:head v=21'12 tt=0'0 tid=20 reqid=client.4540.0:166 client_op=osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~212992]}) 2019-12-07T19:33:58.827+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: starting commit on Op(1:3f7a742d:::smithi13140796-9:head v=21'12 tt=0'0 tid=20 reqid=client.4540.0:166 client_op=osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~212992]}) 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: ExtentCache( Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~212992:12) ) 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] generate_transactions: 1:3f7a742d:::smithi13140796-9:head, current size is 147456 buffers are {0~212992(212992)} 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): adding buffer_update 0,212992 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): to_append: {0~212992(212992)} 2019-12-07T19:33:58.827+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): appending 0~212992 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] encode_and_write: 1:3f7a742d:::smithi13140796-9:head new_size 212992 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): 1:3f7a742d:::smithi13140796-9:head resetting hinfo to logical size 212992 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] operator(): marking append 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: ExtentCache( Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~212992:12) ) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: written: {1:3f7a742d:::smithi13140796-9:head={0~212992(212992)}} 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: op: Op(1:3f7a742d:::smithi13140796-9:head v=21'12 tt=0'0 tid=20 reqid=client.4540.0:166 client_op=osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~212992]}) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: written_set: {1:3f7a742d:::smithi13140796-9:head=[0~212992]} 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] try_reads_to_commit: 1:3f7a742d:::smithi13140796-9:head,{0~212992(212992)} 2019-12-07T19:33:58.828+0000 7f2b7cceb700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=20, reqid=client.4540.0:166, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557e9b9e000 con 0x5557e79cac00 2019-12-07T19:33:58.828+0000 7f2b7cceb700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=20, reqid=client.4540.0:166, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557e9ba0000 con 0x5557e7519800 2019-12-07T19:33:58.828+0000 7f2b7cceb700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=20, reqid=client.4540.0:166, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557e9ba0800 con 0x5557e79ca800 2019-12-07T19:33:58.828+0000 7f2b7cceb700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=20, reqid=client.4540.0:166, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557ea033800 con 0x5557e79cbc00 2019-12-07T19:33:58.828+0000 7f2b7cceb700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=20, reqid=client.4540.0:166, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 -- 0x5557ea034000 con 0x5557e79ca000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 earliest_dup_version = 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 trim 21'12 (21'6) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:166 0.000000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] append_log log((0'0,21'11], crt=21'11) [21'12 (21'6) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:166 0.000000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 lua=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] add_log_entry 21'12 (21'6) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:166 0.000000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 lua=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] append_log approx pg log length = 12 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 lua=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557ea18a300 seq 21 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) get_onode oid 0#1:3f7a742d:::smithi13140796-9:head#c key 0x8080000000000000013f7a74'-!smithi13140796-9!='0xfffffffffffffffe000000000000000c'o' 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.cs0_head 0x5557e75b54a0) r -2 v.len 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# -> 0#1:3f7a742d:::smithi13140796-9:head#c 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# -> 0#1:3f7a742d:::smithi13140796-9:head#c = 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1128 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# = 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 3 keys 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 3 keys = 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 0x0~d000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 0#1:3f7a742d:::smithi13140796-9:head# 0x0~d000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~d000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557ea18a300 7 blobs 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 fbmap_alloc 0x5557e6282e00 allocate 0xd000/1000,d000,0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x19c000~d000/1000,d000,0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x19c000~d000] 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e0000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e00e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e0000 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e00e0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0000) get_ref 0x0~2000 Blob(0x5557ea1e0000 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e00e0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557ea1e0000 blob([0x19c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea1e00e0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.BufferSpace(0x5557ea1e00f8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e0150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e0150 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0230 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0150) get_ref 0x0~2000 Blob(0x5557ea1e0150 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0230 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557ea1e0150 blob([0x19e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea1e0230 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.BufferSpace(0x5557ea1e0248 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e02a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e02a0 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0310 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e02a0) get_ref 0x0~2000 Blob(0x5557ea1e02a0 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0310 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e02a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557ea1e02a0 blob([0x1a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea1e0310 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.BufferSpace(0x5557ea1e0328 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e0380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e03f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e0380 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e03f0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0380) get_ref 0x0~2000 Blob(0x5557ea1e0380 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e03f0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557ea1e0380 blob([0x1a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea1e03f0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.BufferSpace(0x5557ea1e0408 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e0460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e04d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e0460 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e04d0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0460) get_ref 0x0~2000 Blob(0x5557ea1e0460 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e04d0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557ea1e0460 blob([0x1a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea1e04d0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.BufferSpace(0x5557ea1e04e8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e0540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e05b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e0540 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e05b0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0540) get_ref 0x0~2000 Blob(0x5557ea1e0540 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e05b0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557ea1e0540 blob([0x1a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea1e05b0 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.BufferSpace(0x5557ea1e05c8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e0620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e0620 blob([0x1a8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0690 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0620) get_ref 0x0~1000 Blob(0x5557ea1e0620 blob([0x1a8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e0690 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.blob(0x5557ea1e0620) get_ref init 0x1000, 1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~1000: 0x0~1000 Blob(0x5557ea1e0620 blob([0x1a8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557ea1e0690 sbid 0x0)) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.BufferSpace(0x5557ea1e06a8 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0xd000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.extentmap(0x5557e9ff9788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# 0x0~d000 = 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea18a300 cost 95162 (10 ios * 4000 + 55162 bytes) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea18a300 onodes 0x5557e9c7c280,0x5557e9ff9680 shared_blobs 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.extentmap(0x5557e9c7c388) update 0#1:3f7a742d:::smithi13140796-9:head#c 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:3f7a742d:::smithi13140796-9:head#c is 584 (483 bytes onode + 2 bytes spanning blobs + 99 bytes inline extents) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.extentmap(0x5557e9ff9788) update 0#1:3f7a742d:::smithi13140796-9:head# 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore.extentmap(0x5557e9ff9788) update inline shard 133 bytes from 7 extents 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 0#1:3f7a742d:::smithi13140796-9:head# is 623 (484 bytes onode + 2 bytes spanning blobs + 137 bytes inline extents) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea18a300 allocated 0x[19c000~d000] released 0x[] 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 freelist allocate 0x19c000~d000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18a300 prepare 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557ea18a300 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 lua=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] op_applied version 21'12 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] eval_repop repgather(0x5557e9cc05a0 21'12 rep_tid=20 committed?=0 r=0) 2019-12-07T19:33:58.828+0000 7f2b7cceb700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] kick_object_context_blocked 1:3f7a742d:::smithi13140796-9:head requeuing 1 requests 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] requeue_op 0x5557e9bf7ce0 2019-12-07T19:33:58.828+0000 7f2b7cceb700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.828+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18a300 aio_wait 2019-12-07T19:33:58.828+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea18a300 2019-12-07T19:33:58.828+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18a300 io_done 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2107 < min_flush_size 524288 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7c280 had 1 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9ff9680 had 1 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2130 < min_flush_size 524288 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2130 < min_flush_size 524288 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x7909e~852 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x7909e 0x7909e~852 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4909e 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x9e 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x8f0 and padding block with 0x710 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x798f0 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.828+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000132737s (0.000003871s flush + 0.000128866s kv commit) 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.828+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557ea18a300> 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.828+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18a300 kv_submitted 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea18a300 2019-12-07T19:33:58.828+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18a300 finishing 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557ea18a300 onodes 0x5557e9c7c280,0x5557e9ff9680 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e00f8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9c142d0 space 0x5557ea1e00f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.828+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e0248 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9c14360 space 0x5557ea1e0248 0x0~2000 writing nocache) 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e0328 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9c143f0 space 0x5557ea1e0328 0x0~2000 writing nocache) 2019-12-07T19:33:58.828+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e0408 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9c14480 space 0x5557ea1e0408 0x0~2000 writing nocache) 2019-12-07T19:33:58.828+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e04e8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9c14510 space 0x5557ea1e04e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.828+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e05c8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9c145a0 space 0x5557ea1e05c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e06a8 in 0x5557e6269ce0) _finish_write discard buffer(0x5557e9c14630 space 0x5557ea1e06a8 0x0~1000 writing nocache) 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea18a300 done 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.828+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557ea18a300 [] 2019-12-07T19:33:58.828+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.828+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf7ce0 prio 63 cost 0 latency 0.006622 osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.828+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.828+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.828+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op: op osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.828+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.828+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.828+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_object_context: found obc in cache: 0x5557e9b21680 2019-12-07T19:33:58.828+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.828+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(write n=1 w=0) oi: 1:3f7a742d:::smithi13140796-9:head(21'12 client.4540.0:166 dirty|data_digest s 212992 uv 6 dd 578a13d9 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.828+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(21'12 client.4540.0:166 dirty|data_digest s 212992 uv 6 dd 578a13d9 alloc_hint [0 0 0]) 2019-12-07T19:33:58.828+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(21'12 client.4540.0:166 dirty|data_digest s 212992 uv 6 dd 578a13d9 alloc_hint [0 0 0]) 2019-12-07T19:33:58.828+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.829+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf7ce0 finish 2019-12-07T19:33:58.829+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.829+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 67 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c76c00 con 0x5557e79cbc00 2019-12-07T19:33:58.829+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695ce0 prio 196 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.829+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.829+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 25 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9d89400 con 0x5557e79ca000 2019-12-07T19:33:58.829+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695e40 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.829+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.829+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695ce0 prio 196 cost 0 latency 0.000044 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.829+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.829+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.829+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.829+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e7695ce0 finish 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.829+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.829+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.829+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695e40 prio 196 cost 0 latency 0.000058 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.829+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 112 ==== MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=91, reqid=client.4540.0:174, at_version=0'0, trim_to=0'0, roll_forward_to=21'13)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9b9e000 con 0x5557e79cac00 2019-12-07T19:33:58.829+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695b80 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=91, reqid=client.4540.0:174, at_version=0'0, trim_to=0'0, roll_forward_to=21'13)) v2 2019-12-07T19:33:58.829+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.829+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695e40 finish 2019-12-07T19:33:58.829+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=91, reqid=client.4540.0:174, at_version=0'0, trim_to=0'0, roll_forward_to=21'13)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.829+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.829+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.829+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=91, reqid=client.4540.0:174, at_version=0'0, trim_to=0'0, roll_forward_to=21'13)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.829+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.829+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=91, reqid=client.4540.0:174, at_version=0'0, trim_to=0'0, roll_forward_to=21'13)) v2) prio 127 cost 0 e21) pg 0x5557e76ba000 2019-12-07T19:33:58.829+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e7695b80 prio 127 cost 0 latency 0.000045 MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=91, reqid=client.4540.0:174, at_version=0'0, trim_to=0'0, roll_forward_to=21'13)) v2 pg pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] 2019-12-07T19:33:58.829+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] _handle_message: MOSDECSubOpWrite(1.8s5 21/18 ECSubWrite(tid=91, reqid=client.4540.0:174, at_version=0'0, trim_to=0'0, roll_forward_to=21'13)) v2 2019-12-07T19:33:58.829+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log log((0'0,21'13], crt=21'12) [] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] rollforward: entry=21'13 (21'11) delete 1:14fc029d:::smithi13140796-41:head by client.4540.0:174 2019-12-07T19:33:58.823507+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.829+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] append_log approx pg log length = 13 2019-12-07T19:33:58.829+0000 7f2b764de700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.829+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.829+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.829+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9b10f00 seq 25 2019-12-07T19:33:58.829+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head#d onode 0x5557ea011180 txc 0x5557e9b10f00 2019-12-07T19:33:58.829+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head#d 0x0 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b420) put_ref 0x0~2000 Blob(0x5557e9c2b420 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec19d0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b490) put_ref 0x0~2000 Blob(0x5557e9c2b490 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e75c6bd0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b570) put_ref 0x0~2000 Blob(0x5557e9c2b570 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b5e0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b650) put_ref 0x0~2000 Blob(0x5557e9c2b650 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b6c0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b730) put_ref 0x0~2000 Blob(0x5557e9c2b730 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2b7a0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1a40) put_ref 0x0~2000 Blob(0x5557e9ec1a40 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1b20) put_ref 0x0~2000 Blob(0x5557e9ec1b20 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1b90 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1c00) put_ref 0x0~2000 Blob(0x5557e9ec1c00 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1c70 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1ce0) put_ref 0x0~2000 Blob(0x5557e9ec1ce0 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1d50 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1dc0) put_ref 0x0~2000 Blob(0x5557e9ec1dc0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1e30 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1ea0) put_ref 0x0~2000 Blob(0x5557e9ec1ea0 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d37260 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9e5b110) put_ref 0x0~2000 Blob(0x5557e9e5b110 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1f10 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1f80) put_ref 0x0~2000 Blob(0x5557e9ec1f80 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08000 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08070) put_ref 0x0~2000 Blob(0x5557e9f08070 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f080e0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08150) put_ref 0x0~2000 Blob(0x5557e9f08150 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f081c0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08230) put_ref 0x0~2000 Blob(0x5557e9f08230 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f082a0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08310) put_ref 0x0~2000 Blob(0x5557e9f08310 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08380 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f083f0) put_ref 0x0~2000 Blob(0x5557e9f083f0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08460 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f084d0) put_ref 0x0~2000 Blob(0x5557e9f084d0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08540 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f085b0) put_ref 0x0~2000 Blob(0x5557e9f085b0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08620 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08690) put_ref 0x0~2000 Blob(0x5557e9f08690 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08700 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.extentmap(0x5557ea011288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e9c2b420 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ec19d0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2c9000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b420) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec19e8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2c9000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9ec19d0) put 0x5557e9ec19d0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec19e8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e9c2b490 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e75c6bd0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2cb000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b490) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e75c6be8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2cb000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e75c6bd0) put 0x5557e75c6bd0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e75c6be8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x5557e9c2b570 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c2b5e0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2cd000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b570) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b5f8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2cd000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9c2b5e0) put 0x5557e9c2b5e0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b5f8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x5557e9c2b650 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c2b6c0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2cf000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b650) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b6d8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2cf000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9c2b6c0) put 0x5557e9c2b6c0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b6d8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x5557e9c2b730 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c2b7a0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2d1000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9c2b730) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b7b8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2d1000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9c2b7a0) put 0x5557e9c2b7a0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9c2b7b8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x5557e9ec1a40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2d3000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1a40) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1ac8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2d3000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9ec1ab0) put 0x5557e9ec1ab0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1ac8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x5557e9ec1b20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ec1b90 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2d5000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1b20) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1ba8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2d5000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9ec1b90) put 0x5557e9ec1b90 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1ba8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x5557e9ec1c00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ec1c70 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2d7000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1c00) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1c88 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2d7000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9ec1c70) put 0x5557e9ec1c70 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1c88 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x5557e9ec1ce0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ec1d50 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2d9000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1ce0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1d68 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2d9000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9ec1d50) put 0x5557e9ec1d50 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1d68 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x5557e9ec1dc0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ec1e30 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2db000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1dc0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1e48 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2db000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9ec1e30) put 0x5557e9ec1e30 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1e48 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x5557e9ec1ea0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9d37260 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2dd000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1ea0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9d37278 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2dd000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9d37260) put 0x5557e9d37260 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9d37278 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x5557e9e5b110 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ec1f10 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2df000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9e5b110) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1f28 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2df000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9ec1f10) put 0x5557e9ec1f10 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9ec1f28 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x5557e9ec1f80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f08000 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2e1000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9ec1f80) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08018 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2e1000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f08000) put 0x5557e9f08000 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08018 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x5557e9f08070 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f080e0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2e3000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08070) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f080f8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2e3000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f080e0) put 0x5557e9f080e0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f080f8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x5557e9f08150 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f081c0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2e5000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08150) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f081d8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2e5000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f081c0) put 0x5557e9f081c0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f081d8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x5557e9f08230 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f082a0 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2e7000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08230) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f082b8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2e7000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f082a0) put 0x5557e9f082a0 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f082b8 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x5557e9f08310 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f08380 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2e9000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08310) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08398 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2e9000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f08380) put 0x5557e9f08380 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08398 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x5557e9f083f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f08460 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2eb000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f083f0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08478 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2eb000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f08460) put 0x5557e9f08460 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08478 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x5557e9f084d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f08540 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2ed000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f084d0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08558 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2ed000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f08540) put 0x5557e9f08540 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08558 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x5557e9f085b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f08620 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2ef000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f085b0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08638 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2ef000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f08620) put 0x5557e9f08620 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08638 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x5557e9f08690 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f08700 sbid 0x0)) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x2f1000~2000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.blob(0x5557e9f08690) discard_unallocated 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08718 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x2f1000~2000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9f08700) put 0x5557e9f08700 removing self from set 0x5557e76a8a30 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9f08718 in 0x5557e6269960) _clear 2019-12-07T19:33:58.829+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 77 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9ef3800 con 0x5557e79ca800 2019-12-07T19:33:58.829+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head#d = 0 2019-12-07T19:33:58.829+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:33:58.829+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e76958c0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:33:58.829+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.829+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:33:58.829+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10f00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10f00 onodes shared_blobs 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10f00 allocated 0x[] released 0x[2c9000~2a000] 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0x2c9000~2a000 2019-12-07T19:33:58.829+0000 7f2b764de700 10 freelist release 0x2c9000~2a000 2019-12-07T19:33:58.829+0000 7f2b764de700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:33:58.829+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 prepare 2019-12-07T19:33:58.829+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10f00 2019-12-07T19:33:58.829+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 io_done 2019-12-07T19:33:58.829+0000 7f2b764de700 10 osd.6 21 dequeue_op 0x5557e7695b80 finish 2019-12-07T19:33:58.829+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.829+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e76958c0 prio 196 cost 0 latency 0.000050 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.829+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.829+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e76958c0 finish 2019-12-07T19:33:58.829+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1250 < min_flush_size 524288 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea011180 had 1 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x798f0~4f9 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x798f0 0x798f0~4f9 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x498f0 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x8f0 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xde9 and padding block with 0x217 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x79de9 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.829+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000072538s (0.000000078s flush + 0.000072460s kv commit) 2019-12-07T19:33:58.829+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.829+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.829+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10f00> 2019-12-07T19:33:58.829+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.829+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 kv_submitted 2019-12-07T19:33:58.829+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10f00 2019-12-07T19:33:58.829+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10f00 finishing 2019-12-07T19:33:58.829+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10f00 onodes 2019-12-07T19:33:58.829+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10f00 done 2019-12-07T19:33:58.829+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:33:58.829+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10f00 [2c9000~2a000] 2019-12-07T19:33:58.829+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0x2c9000~2a000 2019-12-07T19:33:58.829+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.829+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.829+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 21/18 ECSubWriteReply(tid=91, last_complete=21'13, committed=1, applied=1)) v2 -- 0x5557e9b36e00 con 0x5557e79cac00 2019-12-07T19:33:58.829+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.829+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:33:58.830+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 113 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9b36e00 con 0x5557e79cac00 2019-12-07T19:33:58.830+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e7695600 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.830+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.830+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.830+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.830+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.830+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.830+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695600 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.830+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.830+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e7695600 finish 2019-12-07T19:33:58.830+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.830+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 40 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9d88800 con 0x5557e7519800 2019-12-07T19:33:58.830+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e9d84dc0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.830+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=20, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:3f7a742d:::smithi13140796-9:head v=21'12 tt=0'0 tid=20 reqid=client.4540.0:166 client_op=osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~212992]}) 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 luod=21'11 crt=21'11 lcod 21'10 mlcod 21'10 active+clean] repop_all_committed: repop tid 20 all committed 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] eval_repop repgather(0x5557e9cc05a0 21'12 rep_tid=20 committed?=1 r=0) 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] commit: repgather(0x5557e9cc05a0 21'12 rep_tid=20 committed?=1 r=0) 2019-12-07T19:33:58.830+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] log_op_stats osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 212992 outb 0 lat 0.008456 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] sending reply on osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x5557ea011b80 2019-12-07T19:33:58.830+0000 7f2b744da700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(166 smithi13140796-9 [copy-from ver 6] v21'12 uv6 ondisk = 0) v8 -- 0x5557ea011b80 con 0x5557e6f87800 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.830+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] publish_stats_to_osd 21:29 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] removing repgather(0x5557e9cc05a0 21'12 rep_tid=20 committed?=1 r=0) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] q front is repgather(0x5557e9cc05a0 21'12 rep_tid=20 committed?=1 r=0) 2019-12-07T19:33:58.830+0000 7f2b744da700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] remove_repop repgather(0x5557e9cc05a0 21'12 rep_tid=20 committed?=1 r=0) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] requeue_op 0x5557e9bf7ce0 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 op_wq(4) _enqueue_front OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.830+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] try_finish_rmw: Op(1:3f7a742d:::smithi13140796-9:head v=21'12 tt=0'0 tid=20 reqid=client.4540.0:166 client_op=osd_op(client.4540.0:166 1.cs0 1:3f7a742d:::smithi13140796-9:head [copy-from ver 6 in=55b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'11 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:3f7a742d:::smithi13140796-9:head=[0~212992]}) 2019-12-07T19:33:58.830+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] try_finish_rmw: ExtentCache( Extents(1:3f7a742d:::smithi13140796-9:head)[ Extent(0~212992:12) ) 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] try_reads_to_commit: starting commit on Op(1:3f7a742d:::smithi13140796-9:head v=0'0 tt=0'0 tid=21 reqid=client.4540.0:166 roll_forward_to=21'12 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] try_reads_to_commit: op: Op(1:3f7a742d:::smithi13140796-9:head v=0'0 tt=0'0 tid=21 reqid=client.4540.0:166 roll_forward_to=21'12 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.830+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.cs3 21/18 ECSubWrite(tid=21, reqid=client.4540.0:166, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 -- 0x5557e763e000 con 0x5557e79cac00 2019-12-07T19:33:58.830+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWrite(1.cs4 21/18 ECSubWrite(tid=21, reqid=client.4540.0:166, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 -- 0x5557e763f000 con 0x5557e7519800 2019-12-07T19:33:58.830+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.cs5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:166, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 -- 0x5557e9d82800 con 0x5557e79ca800 2019-12-07T19:33:58.830+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.cs2 21/18 ECSubWrite(tid=21, reqid=client.4540.0:166, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 -- 0x5557e9d81800 con 0x5557e79cbc00 2019-12-07T19:33:58.830+0000 7f2b744da700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.cs1 21/18 ECSubWrite(tid=21, reqid=client.4540.0:166, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 -- 0x5557e9d80000 con 0x5557e79ca000 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'11 lcod 21'11 mlcod 21'11 active+clean] append_log log((0'0,21'12], crt=21'11) [] 2019-12-07T19:33:58.830+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] rollforward: entry=21'12 (21'6) modify 1:3f7a742d:::smithi13140796-9:head by client.4540.0:166 0.000000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] append_log approx pg log length = 12 2019-12-07T19:33:58.830+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.830+0000 7f2b744da700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.830+0000 7f2b744da700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.830+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e9d5bb00 seq 22 2019-12-07T19:33:58.830+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head#c onode 0x5557e9c7c280 txc 0x5557e9d5bb00 2019-12-07T19:33:58.830+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head#c 0x0 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e7621570) put_ref 0x0~2000 Blob(0x5557e7621570 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621500 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e76215e0) put_ref 0x0~2000 Blob(0x5557e76215e0 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621650 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e7621b20) put_ref 0x0~2000 Blob(0x5557e7621b20 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e76219d0 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e7621c00) put_ref 0x0~2000 Blob(0x5557e7621c00 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621a40 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e7621ab0) put_ref 0x0~1000 Blob(0x5557e7621ab0 blob([0x6e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9be2d20 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.extentmap(0x5557e9c7c388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e7621570 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7621500 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x66000~2000] 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e7621570) discard_unallocated 0x0~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621518 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 _rm buffer(0x5557e79ab440 space 0x5557e7621518 0x0~1000 clean) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x66000~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.sharedblob(0x5557e7621500) put 0x5557e7621500 removing self from set 0x5557e75b5570 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621518 in 0x5557e6269ce0) _clear 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e76215e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7621650 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x68000~2000] 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e76215e0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621668 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x68000~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.sharedblob(0x5557e7621650) put 0x5557e7621650 removing self from set 0x5557e75b5570 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621668 in 0x5557e6269ce0) _clear 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x5557e7621b20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e76219d0 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x6a000~2000] 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e7621b20) discard_unallocated 0x0~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e76219e8 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x6a000~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.sharedblob(0x5557e76219d0) put 0x5557e76219d0 removing self from set 0x5557e75b5570 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e76219e8 in 0x5557e6269ce0) _clear 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x5557e7621c00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7621a40 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x6c000~2000] 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e7621c00) discard_unallocated 0x0~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621a58 in 0x5557e6269ce0) _discard 0x0~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x6c000~2000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.sharedblob(0x5557e7621a40) put 0x5557e7621a40 removing self from set 0x5557e75b5570 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e7621a58 in 0x5557e6269ce0) _clear 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x8000~1000: 0x0~1000 Blob(0x5557e7621ab0 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x5557e9be2d20 sbid 0x0)) 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x6e000~1000] 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.blob(0x5557e7621ab0) discard_unallocated 0x0~1000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9be2d38 in 0x5557e6269ce0) _discard 0x0~1000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x6e000~1000 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.sharedblob(0x5557e9be2d20) put 0x5557e9be2d20 removing self from set 0x5557e75b5570 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore.BufferSpace(0x5557e9be2d38 in 0x5557e6269ce0) _clear 2019-12-07T19:33:58.830+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.cs0_head 0#1:3f7a742d:::smithi13140796-9:head#c = 0 2019-12-07T19:33:58.830+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.830+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.830+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:33:58.830+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5bb00 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:33:58.831+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5bb00 onodes shared_blobs 2019-12-07T19:33:58.831+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5bb00 allocated 0x[] released 0x[66000~9000] 2019-12-07T19:33:58.831+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0x66000~9000 2019-12-07T19:33:58.831+0000 7f2b744da700 10 freelist release 0x66000~9000 2019-12-07T19:33:58.831+0000 7f2b744da700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.831+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 prepare 2019-12-07T19:33:58.831+0000 7f2b744da700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5bb00 2019-12-07T19:33:58.831+0000 7f2b744da700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 io_done 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d84dc0 finish 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1249 < min_flush_size 524288 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7c280 had 1 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x79de9~4f8 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x79de9 0x79de9~4f8 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x49de9 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xde9 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x2e1 and padding block with 0xd1f 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x7a2e1 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:33:58.831+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000140143s (0.000000064s flush + 0.000140079s kv commit) 2019-12-07T19:33:58.831+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.831+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:33:58.831+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5bb00> 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf7ce0 prio 63 cost 0 latency 0.008970 osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] 2019-12-07T19:33:58.831+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.831+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 kv_submitted 2019-12-07T19:33:58.831+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5bb00 2019-12-07T19:33:58.831+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5bb00 finishing 2019-12-07T19:33:58.831+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5bb00 onodes 2019-12-07T19:33:58.831+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5bb00 done 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.831+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:33:58.831+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5bb00 [66000~9000] 2019-12-07T19:33:58.831+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0x66000~9000 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] _handle_message: osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.831+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:33:58.831+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] do_op: op osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] op_has_sufficient_caps session=0x5557e6295860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] do_op osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.831+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 41 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9ef2400 con 0x5557e7519800 2019-12-07T19:33:58.831+0000 7f2b95fb3700 15 osd.6 21 enqueue_op 0x5557e9d854a0 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b95fb3700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] get_object_context: found obc in cache: 0x5557e9b21680 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] get_object_context: 0x5557e9b21680 1:3f7a742d:::smithi13140796-9:head rwstate(none n=0 w=0) oi: 1:3f7a742d:::smithi13140796-9:head(21'12 client.4540.0:166 dirty|data_digest s 212992 uv 6 dd 578a13d9 alloc_hint [0 0 0]) exists: 1 ssc: 0x5557e79679e0 snapset: 0=[]:{} 2019-12-07T19:33:58.831+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 114 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c76000 con 0x5557e79cac00 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] find_object_context 1:3f7a742d:::smithi13140796-9:head @head oi=1:3f7a742d:::smithi13140796-9:head(21'12 client.4540.0:166 dirty|data_digest s 212992 uv 6 dd 578a13d9 alloc_hint [0 0 0]) 2019-12-07T19:33:58.831+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5a20 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.831+0000 7f2b784e2700 25 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] do_op oi 1:3f7a742d:::smithi13140796-9:head(21'12 client.4540.0:166 dirty|data_digest s 212992 uv 6 dd 578a13d9 alloc_hint [0 0 0]) 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] do_op obc obc(1:3f7a742d:::smithi13140796-9:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] execute_ctx 0x5557e9b13c00 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] execute_ctx 1:3f7a742d:::smithi13140796-9:head [stat] ov 21'12 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] do_osd_op 1:3f7a742d:::smithi13140796-9:head [stat] 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] do_osd_op stat 2019-12-07T19:33:58.831+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 78 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9ef3a00 con 0x5557e79ca800 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] stat oi has 212992 2019-12-07T19:33:58.638318+0000 2019-12-07T19:33:58.831+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c46e0 prio 196 cost 0 latency 0.000009 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] execute_ctx alloc reply 0x5557e9d59400 result 0 2019-12-07T19:33:58.831+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] do_osd_op_effects client.4540 con 0x5557e6f87800 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.831+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] log_op_stats osd_op(client.4540.0:167 1.cs0 1:3f7a742d:::smithi13140796-9:head [stat out=16b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 16 lat 0.009122 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.831+0000 7f2b784e2700 15 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] publish_stats_to_osd 21:30 2019-12-07T19:33:58.831+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/224485236 -- osd_op_reply(167 smithi13140796-9 [stat out=16b] v0'0 uv6 ondisk = 0) v8 -- 0x5557e9d59400 con 0x5557e6f87800 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e9bf7ce0 finish 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 68 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (crc 0 0 0) 0x5557e9c76600 con 0x5557e79cbc00 2019-12-07T19:33:58.831+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c4160 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.831+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.831+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.5 v2:172.21.15.109:6802/36936 26 ==== MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x5557e9ef3600 con 0x5557e79ca000 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d854a0 prio 196 cost 0 latency 0.000165 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] 2019-12-07T19:33:58.831+0000 7f2b957b2700 15 osd.6 21 enqueue_op 0x5557e75c5ce0 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b957b2700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e9d854a0 finish 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c46e0 prio 196 cost 0 latency 0.000133 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c46e0 finish 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5a20 prio 196 cost 0 latency 0.000202 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5a20 finish 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4160 prio 196 cost 0 latency 0.000122 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b784e2700 10 osd.6 21 dequeue_op 0x5557e75c4160 finish 2019-12-07T19:33:58.831+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGOpItem(op=MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x5557e6e89000 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5ce0 prio 196 cost 0 latency 0.000125 MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 pg pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] _handle_message: MOSDECSubOpWriteReply(1.cs0 21/18 ECSubWriteReply(tid=21, last_complete=21'12, committed=1, applied=1)) v2 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] try_finish_rmw: Op(1:3f7a742d:::smithi13140796-9:head v=0'0 tt=0'0 tid=21 reqid=client.4540.0:166 roll_forward_to=21'12 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:33:58.831+0000 7f2b744da700 10 osd.6 21 dequeue_op 0x5557e75c5ce0 finish 2019-12-07T19:33:58.831+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.844+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.894+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.943+0000 7f2b937ae700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 6 (PID: 36932) UID: 0 2019-12-07T19:33:58.944+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.995+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.044+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.095+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.145+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.195+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.245+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.295+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.345+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.371+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.7 172.21.15.109:0/36939 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.690547135s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e765e400 con 0x5557e748f800 2019-12-07T19:33:59.371+0000 7f2b957b2700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690290739s,18446744073.690547135s]) 2019-12-07T19:33:59.371+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.827435739s delta_ub 0.019260877s) v5 -- 0x5557e76b1c00 con 0x5557e748f800 2019-12-07T19:33:59.371+0000 7f2b957b2700 10 osd.6 21 maybe_share_map con 0x5557e79f4000 v2:172.21.15.109:6826/36939 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:59.371+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.7 172.21.15.109:0/36939 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.690547135s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e765e400 con 0x5557e74cc800 2019-12-07T19:33:59.371+0000 7f2b957b2700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690240427s,18446744073.690547135s]) 2019-12-07T19:33:59.371+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.827486051s delta_ub 0.019311189s) v5 -- 0x5557e7a9a600 con 0x5557e74cc800 2019-12-07T19:33:59.395+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.445+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.450+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.1 172.21.15.131:0/37078 5 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.908345690s delta_ub 0.002860236s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e794ee00 con 0x5557e7518400 2019-12-07T19:33:59.450+0000 7f2b967b4700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002271309s,0.002860236s]) 2019-12-07T19:33:59.450+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.906074381s delta_ub 18446744073.707280307s) v5 -- 0x5557e9d89c00 con 0x5557e7518400 2019-12-07T19:33:59.450+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.1 172.21.15.131:0/37078 5 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.908345690s delta_ub 0.002860236s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9d89800 con 0x5557e74cdc00 2019-12-07T19:33:59.450+0000 7f2b967b4700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002205162s,0.002860236s]) 2019-12-07T19:33:59.450+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.906140528s delta_ub 18446744073.707346454s) v5 -- 0x5557e9d89c00 con 0x5557e74cdc00 2019-12-07T19:33:59.495+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.545+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.574+0000 7f2b9171a700 10 osd.6 21 tick_without_osd_lock 2019-12-07T19:33:59.574+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fbdb000/0x40000000/0x1680000000, data 0x3ffda8/0x423000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:59.574+0000 7f2b9171a700 25 osd.6 21 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055010+0000 last_rx_front 2019-12-07T19:33:57.055094+0000 2019-12-07T19:33:59.574+0000 7f2b9171a700 25 osd.6 21 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055240+0000 last_rx_front 2019-12-07T19:33:57.055135+0000 2019-12-07T19:33:59.574+0000 7f2b9171a700 25 osd.6 21 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055116+0000 last_rx_front 2019-12-07T19:33:57.055052+0000 2019-12-07T19:33:59.574+0000 7f2b9171a700 25 osd.6 21 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055209+0000 last_rx_front 2019-12-07T19:33:57.055220+0000 2019-12-07T19:33:59.574+0000 7f2b9171a700 25 osd.6 21 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055145+0000 last_rx_front 2019-12-07T19:33:57.055175+0000 2019-12-07T19:33:59.574+0000 7f2b9171a700 25 osd.6 21 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055064+0000 last_rx_front 2019-12-07T19:33:57.055188+0000 2019-12-07T19:33:59.574+0000 7f2b9171a700 25 osd.6 21 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055254+0000 last_rx_front 2019-12-07T19:33:57.055160+0000 2019-12-07T19:33:59.574+0000 7f2b9171a700 20 osd.6 21 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:59.574+0000 7f2b9171a700 20 osd.6 21 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:59.575+0000 7f2b9171a700 20 osd.6 21 scrub_load_below_threshold loadavg per cpu 0.0625 < max 5 = yes 2019-12-07T19:33:59.575+0000 7f2b9171a700 20 osd.6 21 sched_scrub load_is_low=1 2019-12-07T19:33:59.575+0000 7f2b9171a700 10 osd.6 21 sched_scrub 1.cs0 scheduled at 2019-12-07T19:35:16.655986+0000 > 2019-12-07T19:33:59.576189+0000 2019-12-07T19:33:59.575+0000 7f2b9171a700 20 osd.6 21 sched_scrub done 2019-12-07T19:33:59.575+0000 7f2b9171a700 10 osd.6 21 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:59.575+0000 7f2b9171a700 20 osd.6 21 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:59.575+0000 7f2b9171a700 10 osd.6 21 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 tick 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 maybe_update_heartbeat_peers forcing update after 20.069501 seconds 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 reset_heartbeat_peers 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 maybe_update_heartbeat_peers updating 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 adding neighbor peer osd.0 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 adding neighbor peer osd.5 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 adding neighbor peer osd.7 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 adding random peer osd.1 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 adding random peer osd.2 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 adding random peer osd.3 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 adding random peer osd.4 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 maybe_update_heartbeat_peers 7 peers, extras 0,1,2,3,4,5,7 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 do_waiters -- start 2019-12-07T19:33:59.591+0000 7f2b92fad700 10 osd.6 21 do_waiters -- finish 2019-12-07T19:33:59.591+0000 7f2b92fad700 20 osd.6 21 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:33:59.596+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.646+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.696+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.746+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.751+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.209920745s delta_ub 0.003175198s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e765e600 con 0x5557e79f4c00 2019-12-07T19:33:59.751+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.209920745s delta_ub 0.003175198s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e794ee00 con 0x5557e79f4800 2019-12-07T19:33:59.751+0000 7f2b957b2700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002744848s,0.003175198s]) 2019-12-07T19:33:59.751+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.207175897s delta_ub 18446744073.706806768s) v5 -- 0x5557e7a9a600 con 0x5557e79f4c00 2019-12-07T19:33:59.751+0000 7f2b967b4700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002721706s,0.003175198s]) 2019-12-07T19:33:59.751+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.207199039s delta_ub 18446744073.706829910s) v5 -- 0x5557e9d89c00 con 0x5557e79f4800 2019-12-07T19:33:59.785+0000 7f2b937ae700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 6 (PID: 36932) UID: 0 2019-12-07T19:33:59.796+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.846+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.858+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.4 172.21.15.109:0/36938 7 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.308927630s delta_ub 18446744073.704628772s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9d89800 con 0x5557e7519000 2019-12-07T19:33:59.858+0000 7f2b967b4700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704030470s,18446744073.704628772s]) 2019-12-07T19:33:59.858+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.4 172.21.15.109:0/36938 7 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.308927630s delta_ub 18446744073.704628772s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9ef2000 con 0x5557e74cc400 2019-12-07T19:33:59.858+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.314448776s delta_ub 0.005521146s) v5 -- 0x5557e9d89c00 con 0x5557e7519000 2019-12-07T19:33:59.858+0000 7f2b95fb3700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.704003010s,18446744073.704628772s]) 2019-12-07T19:33:59.858+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.314476236s delta_ub 0.005548606s) v5 -- 0x5557e9b37400 con 0x5557e74cc400 2019-12-07T19:33:59.896+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.946+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.996+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.046+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.097+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.147+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.197+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.247+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 384554 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.251+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.3 172.21.15.131:0/37076 5 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.710352893s delta_ub 0.003282298s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e765e400 con 0x5557e79f4c00 2019-12-07T19:34:00.251+0000 7f2b957b2700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002845448s,0.003282298s]) 2019-12-07T19:34:00.251+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.707507445s delta_ub 18446744073.706706168s) v5 -- 0x5557e7a9a600 con 0x5557e79f4c00 2019-12-07T19:34:00.251+0000 7f2b957b2700 10 osd.6 21 maybe_share_map con 0x5557e79ca800 v2:172.21.15.131:6812/37076 map epoch 21 -> 22 (as per caller) 2019-12-07T19:34:00.251+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.3 172.21.15.131:0/37076 5 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.710352893s delta_ub 0.003282298s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e794ee00 con 0x5557e79f4800 2019-12-07T19:34:00.251+0000 7f2b967b4700 20 osd.6 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002808455s,0.003282298s]) 2019-12-07T19:34:00.251+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.707544438s delta_ub 18446744073.706743161s) v5 -- 0x5557e9d89c00 con 0x5557e79f4800 2019-12-07T19:34:00.252+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 79 ==== osd_map(22..22 src has 1..22) v4 ==== 633+0+0 (crc 0 0 0) 0x5557e9c13200 con 0x5557e79ca800 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 osd.6 21 OSD::ms_dispatch: osd_map(22..22 src has 1..22) v4 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 osd.6 21 do_waiters -- start 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 osd.6 21 do_waiters -- finish 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 osd.6 21 _dispatch 0x5557e9c13200 osd_map(22..22 src has 1..22) v4 2019-12-07T19:34:00.252+0000 7f2b834f8700 3 osd.6 21 handle_osd_map epochs [22,22], i have 21, src has [1,22] 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 osd.6 21 handle_osd_map got inc map for epoch 22 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 osd.6 21 got_full_map 22, nothing requested 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 osd.6 21 handle_osd_map pg_num_history pg_num_history(e22 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps purged_snaps {22={}} 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 osd.6 21 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e22 [1,22] lci=[13,22]) 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557e7a93500 seq 13 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c89b972f:::inc_osdmap.22:0# key 0x7f7fffffffffffffffc89b97'/!inc_osdmap.22!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:34:00.252+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1129 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24d 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a93500 1 blobs 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x66000~1000/1000,1000,0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x66000~1000] 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d37490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d37490 blob([0x66000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2310 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e9d37490) get_ref 0x0~24d Blob(0x5557e9d37490 blob([0x66000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9be2310 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e9d37490) get_ref init 0x1000, 1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x5557e9d37490 blob([0x66000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x5557e9be2310 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e9be2328 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x24d 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557788) dirty_range mark inline shard dirty 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d = 0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3760e826:::osdmap.22:0# key 0x7f7fffffffffffffff3760e8'&!osdmap.22!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:34:00.252+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3760e826:::osdmap.22:0# 0x0~140c 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1130 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3760e826:::osdmap.22:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~40c 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a93500 2 blobs 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x67000~2000/1000,2000,0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x67000~2000] 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d369a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d367e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d369a0 blob([0x67000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d367e0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e9d369a0) get_ref 0x0~1000 Blob(0x5557e9d369a0 blob([0x67000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d367e0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e9d369a0) get_ref init 0x1000, 1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e9d369a0 blob([0x67000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9d367e0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e9d367f8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d92000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d92000 blob([!~1000,0x68000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56c40 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e9d92000) get_ref 0x1000~40c Blob(0x5557e9d92000 blob([!~1000,0x68000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e9b56c40 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e9d92000) get_ref init 0x2000, 1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x5557e9d92000 blob([!~1000,0x68000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x5557e9b56c40 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e9b56c58 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x140c 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557a08) dirty_range mark inline shard dirty 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3760e826:::osdmap.22:0# 0x0~140c = 0 2019-12-07T19:34:00.252+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~26 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a93500 0 blobs 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:00.252+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.252+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.252+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e7a93500 0 blobs 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93500 cost 23202 (4 ios * 4000 + 7202 bytes) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93500 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7557680,0x5557e7557900 shared_blobs 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557788) update #-1:c89b972f:::inc_osdmap.22:0# 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557788) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c89b972f:::inc_osdmap.22:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557a08) update #-1:3760e826:::osdmap.22:0# 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e7557a08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3760e826:::osdmap.22:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93500 allocated 0x[66000~3000] released 0x[] 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 freelist allocate 0x66000~3000 2019-12-07T19:34:00.252+0000 7f2b834f8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93500 prepare 2019-12-07T19:34:00.252+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e7a93500 2019-12-07T19:34:00.252+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93500 aio_wait 2019-12-07T19:34:00.252+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93500 2019-12-07T19:34:00.252+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93500 io_done 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7557680 had 1 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7557900 had 1 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs get_usage bdev 1 free 3860332544 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluestore(/var/lib/ceph/osd/ceph-6) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998661) bluestore 86 GiB free (0.959955), bluefs_ratio 0.0399508 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x7a2e1~226c to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x7a2e1 0x7a2e1~226c to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4a2e1 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x2e1 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x54d and padding block with 0xab3 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x7c54d 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.252+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000146608s (0.000004639s flush + 0.000141969s kv commit) 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.252+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7a93500> 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.252+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93500 kv_submitted 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93500 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557e7a93500 osr 0x5557e76ce500 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e9b5adc0) _discard drop 10 0x1b000~1000 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6 21 _committed_osd_maps 22..22 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6 21 advance to epoch 22 (<= last 22 <= newest_map 22) 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e9b5adc0) prepare_write seq 11 0x1b000~1000 crc 2e955191 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e9b5adc0) _discard drop 10 0x2000~1000 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557e9b5adc0) prepare_write seq 11 0x2000~1000 crc 46e4b91e 2019-12-07T19:34:00.252+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.252+0000 7f2b85cfd700 7 osd.6 22 consume_map version 22 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.8s5 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.1s2 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.9s5 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.2s3 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.as1 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.bs1 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.3s5 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.cs0 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.5s1 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.6s1 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.es5 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.7s1 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 identify_splits_and_merges 1.fs5 e21 to e22 pg_nums {18=16} 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 22 prune_pg_created 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6:0.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:0.consume_map consume_map 1.8s5 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6:1.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.1s2 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.9s5 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6:2.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.2s3 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.as1 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6:3.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.bs1 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.3s5 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6:4.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:4.consume_map consume_map 1.cs0 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6:5.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:5.consume_map consume_map 1.5s1 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6:6.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.6s1 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.es5 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6:7.consume_map 22 (was 21) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.7s1 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.fs5 2019-12-07T19:34:00.252+0000 7f2b85cfd700 10 osd.6 22 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:00.252+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.8s5 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.1s2 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.9s5 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.2s3 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.as1 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f2b85cfd700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e76ba000 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e767f000 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e769f000 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6:0.update_pg_epoch 1.8s5 21 -> 22 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 21 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] handle_advance_map [1,4,5,2,3,6]/[1,4,5,2,3,6] -- 1/1 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6:2.update_pg_epoch 1.2s3 21 -> 22 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6:1.update_pg_epoch 1.1s2 21 -> 22 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 osd.6 pg_epoch: 21 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.8s5_head options 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.2s3_head options 2019-12-07T19:34:00.252+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.1s2_head options 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] agent_stop 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] agent_stop 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] agent_stop 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] activate_map 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] activate_map 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.252+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.252+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e767c000 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6:1.update_pg_epoch 1.9s5 21 -> 22 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 21 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.252+0000 7f2b79ce5700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.9s5_head options 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.252+0000 7f2b79ce5700 20 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:34:00.252+0000 7f2b79ce5700 10 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.bs1 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b85cfd700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.3s5 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b85cfd700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e766d000 2019-12-07T19:34:00.253+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.cs0 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b85cfd700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.253+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.5s1 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b85cfd700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.253+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.6s1 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b85cfd700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.253+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.es5 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b85cfd700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6:2.update_pg_epoch 1.as1 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e769c000 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e6e89000 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 osd.6 pg_epoch: 21 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6:4.update_pg_epoch 1.cs0 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6:6.update_pg_epoch 1.6s1 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.as1_head options 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 osd.6 pg_epoch: 21 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 21 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] handle_advance_map [6,5,4,1,2,3]/[6,5,4,1,2,3] -- 6/6 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] state: Active advmap 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.6s1_head options 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e7957000 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.cs0_head options 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] agent_stop 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6:3.update_pg_epoch 1.bs1 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:34:00.253+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] activate_map 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] state: Active: handling ActMap 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] Active: kicking snap trim 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e764c000 2019-12-07T19:34:00.253+0000 7f2b784e2700 7 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] state: handle ActMap primary 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.bs1_head options 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.253+0000 7f2b784e2700 15 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] publish_stats_to_osd 21:31 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6:5.update_pg_epoch 1.5s1 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 osd.6 pg_epoch: 21 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [2,6,3,4,1,5]/[2,6,3,4,1,5] -- 2/2 2019-12-07T19:34:00.253+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.7s1 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b85cfd700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b85cfd700 15 osd.6 22 enqueue_peering_evt 1.fs5 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b85cfd700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e76bd000 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.5s1_head options 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e766a000 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e7954000 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6:7.update_pg_epoch 1.7s1 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6:6.update_pg_epoch 1.es5 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 osd.6 pg_epoch: 21 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 osd.6 pg_epoch: 21 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6:3.update_pg_epoch 1.3s5 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 21 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b774e0700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.es5_head options 2019-12-07T19:34:00.253+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.7s1_head options 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b73cd9700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.3s5_head options 2019-12-07T19:34:00.253+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f2b85cfd700 7 osd.6 22 activate_map version 22 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:34:00.253+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.253+0000 7f2b7ace7700 20 osd.6 22 agent_entry empty queue 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x5557e764f000 2019-12-07T19:34:00.253+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 22 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6:7.update_pg_epoch 1.fs5 21 -> 22 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 osd.6 pg_epoch: 21 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.253+0000 7f2b76cdf700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.fs5_head options 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 22 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 22 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:34:00.253+0000 7f2b76cdf700 10 log is not dirty 2019-12-07T19:34:00.253+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.297+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 388190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.347+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 388190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.397+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 388190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.441+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.5 172.21.15.109:0/36936 7 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.909301593s delta_ub 0.012172630s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e7a9b400 con 0x5557e62f4c00 2019-12-07T19:34:00.441+0000 7f2b95fb3700 20 osd.6 22 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011646672s,0.012172630s]) 2019-12-07T19:34:00.441+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.897654921s delta_ub 18446744073.697904944s) v5 -- 0x5557e9b37400 con 0x5557e62f4c00 2019-12-07T19:34:00.441+0000 7f2b95fb3700 10 osd.6 22 maybe_share_map con 0x5557e79ca000 v2:172.21.15.109:6802/36936 map epoch 21 -> 22 (as per caller) 2019-12-07T19:34:00.441+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.5 172.21.15.109:0/36936 7 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.909301593s delta_ub 0.012172630s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e9d89800 con 0x5557e7518c00 2019-12-07T19:34:00.441+0000 7f2b967b4700 20 osd.6 22 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011612886s,0.012172630s]) 2019-12-07T19:34:00.441+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.897688707s delta_ub 18446744073.697938730s) v5 -- 0x5557e9d89c00 con 0x5557e7518c00 2019-12-07T19:34:00.447+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 388190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.497+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 388190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.520+0000 7f2b83cf9700 20 osd.6 22 reports for 0 queries 2019-12-07T19:34:00.520+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.6 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x5557ea244000 con 0x5557e7a48000 2019-12-07T19:34:00.520+0000 7f2b83cf9700 20 bluestore(/var/lib/ceph/osd/ceph-6) pool_statfs pool 1 2019-12-07T19:34:00.520+0000 7f2b83cf9700 10 bluestore(/var/lib/ceph/osd/ceph-6) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x3ee000/0x3ee000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:00.520+0000 7f2b83cf9700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(1 pgs tid 0 v 0) v2 -- 0x5557ea244380 con 0x5557e7a48000 2019-12-07T19:34:00.547+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 388190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.576+0000 7f2b92fad700 10 osd.6 22 tick 2019-12-07T19:34:00.576+0000 7f2b92fad700 10 osd.6 22 do_waiters -- start 2019-12-07T19:34:00.576+0000 7f2b92fad700 10 osd.6 22 do_waiters -- finish 2019-12-07T19:34:00.576+0000 7f2b92fad700 20 osd.6 22 tick last_purged_snaps_scrub 2019-12-07T19:33:37.564652+0000 next 2019-12-07T19:34:37.564652+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 10 osd.6 22 tick_without_osd_lock 2019-12-07T19:34:00.582+0000 7f2b9171a700 20 bluestore(/var/lib/ceph/osd/ceph-6) statfs store_statfs(0x163fbd8000/0x40000000/0x1680000000, data 0x401401/0x426000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:00.582+0000 7f2b9171a700 25 osd.6 22 heartbeat_check osd.0 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055010+0000 last_rx_front 2019-12-07T19:33:57.055094+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 25 osd.6 22 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055240+0000 last_rx_front 2019-12-07T19:33:57.055135+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 25 osd.6 22 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055116+0000 last_rx_front 2019-12-07T19:33:57.055052+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 25 osd.6 22 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055209+0000 last_rx_front 2019-12-07T19:33:57.055220+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 25 osd.6 22 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055145+0000 last_rx_front 2019-12-07T19:33:57.055175+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 25 osd.6 22 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055064+0000 last_rx_front 2019-12-07T19:33:57.055188+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 25 osd.6 22 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.253160+0000 last_tx 2019-12-07T19:33:57.054652+0000 last_rx_back 2019-12-07T19:33:57.055254+0000 last_rx_front 2019-12-07T19:33:57.055160+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 20 osd.6 22 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:00.582+0000 7f2b9171a700 20 osd.6 22 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:00.582+0000 7f2b9171a700 20 osd.6 22 scrub_load_below_threshold loadavg per cpu 0.0625 < max 5 = yes 2019-12-07T19:34:00.582+0000 7f2b9171a700 20 osd.6 22 sched_scrub load_is_low=1 2019-12-07T19:34:00.582+0000 7f2b9171a700 10 osd.6 22 sched_scrub 1.cs0 scheduled at 2019-12-07T19:35:16.655986+0000 > 2019-12-07T19:34:00.583359+0000 2019-12-07T19:34:00.582+0000 7f2b9171a700 20 osd.6 22 sched_scrub done 2019-12-07T19:34:00.582+0000 7f2b9171a700 10 osd.6 22 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:00.582+0000 7f2b9171a700 20 osd.6 22 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:00.582+0000 7f2b9171a700 10 osd.6 22 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:00.598+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:34:00.598+0000 7f2b8ad07700 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock osr 0x5557e76ce500 2 ios pending 2019-12-07T19:34:00.598+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock seq 11 0x2000~1000 2019-12-07T19:34:00.598+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock write 0x2000~1000 crc 46e4b91e 2019-12-07T19:34:00.598+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock seq 11 0x1b000~1000 2019-12-07T19:34:00.598+0000 7f2b8ad07700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_submit_unlock write 0x1b000~1000 crc 2e955191 2019-12-07T19:34:00.598+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 388190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.598+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_aio_finish osr 0x5557e76ce500 2019-12-07T19:34:00.598+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_aio_finish dequeueing 2019-12-07T19:34:00.615+0000 7f2b967b4700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e7450400 0x5557e7a99180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:00.615+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> conn(0x5557e7450400 0x5557e7a99180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:00.615+0000 7f2b957b2700 10 osd.6 22 ms_handle_authentication new session 0x5557e9c2e4e0 con 0x5557e7450400 entity client.admin addr 2019-12-07T19:34:00.615+0000 7f2b957b2700 10 osd.6 22 ms_handle_authentication session 0x5557e9c2e4e0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:00.615+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/890522420 conn(0x5557e7450400 0x5557e7a99180 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).ready entity=client.4352 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:00.615+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4352 172.21.15.131:0/890522420 1 ==== command(tid 13: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x5557e75e3860 con 0x5557e7450400 2019-12-07T19:34:00.615+0000 7f2b88017700 20 osd.6 22 OSD::ms_dispatch: command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:00.615+0000 7f2b88017700 10 osd.6 22 do_waiters -- start 2019-12-07T19:34:00.615+0000 7f2b88017700 10 osd.6 22 do_waiters -- finish 2019-12-07T19:34:00.615+0000 7f2b88017700 20 osd.6 22 _dispatch 0x5557e75e3860 command(tid 13: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:00.616+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/890522420 -- command_reply(tid 13: 0 ) v1 -- 0x5557e627f340 con 0x5557e7450400 2019-12-07T19:34:00.636+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 69 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79cbc00 2019-12-07T19:34:00.636+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.bs1 epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:34:00.636+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.636+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.636+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.636+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 70 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79cbc00 2019-12-07T19:34:00.636+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.636+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) pg 0x5557e7957000 2019-12-07T19:34:00.636+0000 7f2b74cdb700 20 osd.6 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 23, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.636+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.636+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.3s5 epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:34:00.636+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.636+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.636+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.636+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.636+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) pg 0x5557e7954000 2019-12-07T19:34:00.636+0000 7f2b78ce3700 20 osd.6 op_wq _add_slot_waiter 1.3s5 peering, item epoch is 23, will wait on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.636+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.639+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 80 ==== pg_info2(1.1s2 1.1s2 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79ca800 2019-12-07T19:34:00.639+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.1s2 epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:34:00.639+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) pg 0x5557e769f000 2019-12-07T19:34:00.639+0000 7f2b75cdd700 20 osd.6 op_wq _add_slot_waiter 1.1s2 peering, item epoch is 23, will wait on OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.639+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.639+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 81 ==== pg_info2(1.6s1 1.6s1 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79ca800 2019-12-07T19:34:00.639+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.6s1 epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:34:00.639+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.640+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.640+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.640+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.640+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) pg 0x5557e769c000 2019-12-07T19:34:00.640+0000 7f2b734d8700 20 osd.6 op_wq _add_slot_waiter 1.6s1 peering, item epoch is 23, will wait on OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.640+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.640+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 82 ==== pg_info2(1.fs5 1.fs5 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79ca800 2019-12-07T19:34:00.640+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.fs5 epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:34:00.640+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.640+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.640+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.640+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.640+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) pg 0x5557e764f000 2019-12-07T19:34:00.640+0000 7f2b72cd7700 20 osd.6 op_wq _add_slot_waiter 1.fs5 peering, item epoch is 23, will wait on OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.640+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.640+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 115 ==== pg_info2(1.9s5 1.9s5 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79cac00 2019-12-07T19:34:00.640+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.9s5 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:34:00.640+0000 7f2b957b2700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 116 ==== pg_info2(1.es5 1.es5 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79cac00 2019-12-07T19:34:00.641+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.es5 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:34:00.641+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) pg 0x5557e766a000 2019-12-07T19:34:00.641+0000 7f2b774e0700 20 osd.6 op_wq _add_slot_waiter 1.es5 peering, item epoch is 23, will wait on OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.641+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) pg 0x5557e767c000 2019-12-07T19:34:00.641+0000 7f2b79ce5700 20 osd.6 op_wq _add_slot_waiter 1.9s5 peering, item epoch is 23, will wait on OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.641+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 117 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79cac00 2019-12-07T19:34:00.641+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.8s5 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s) 2019-12-07T19:34:00.641+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e23) pg 0x5557e76ba000 2019-12-07T19:34:00.641+0000 7f2b764de700 20 osd.6 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 23, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.641+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 118 ==== pg_info2(1.7s1 1.7s1 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79cac00 2019-12-07T19:34:00.641+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.7s1 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:34:00.641+0000 7f2b957b2700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) pg 0x5557e76bd000 2019-12-07T19:34:00.641+0000 7f2b76cdf700 20 osd.6 op_wq _add_slot_waiter 1.7s1 peering, item epoch is 23, will wait on OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.642+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 119 ==== pg_info2(1.as1 1.as1 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9b13500 con 0x5557e79cac00 2019-12-07T19:34:00.642+0000 7f2b957b2700 15 osd.6 22 enqueue_peering_evt 1.as1 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:34:00.642+0000 7f2b957b2700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.642+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.642+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.642+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.642+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) pg 0x5557e766d000 2019-12-07T19:34:00.642+0000 7f2b794e4700 20 osd.6 op_wq _add_slot_waiter 1.as1 peering, item epoch is 23, will wait on OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.642+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.643+0000 7f2b88017700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] <== client.4352 172.21.15.131:0/890522420 2 ==== command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x5557e75e3a40 con 0x5557e7450400 2019-12-07T19:34:00.643+0000 7f2b88017700 20 osd.6 22 OSD::ms_dispatch: command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:00.643+0000 7f2b88017700 10 osd.6 22 do_waiters -- start 2019-12-07T19:34:00.643+0000 7f2b88017700 10 osd.6 22 do_waiters -- finish 2019-12-07T19:34:00.643+0000 7f2b88017700 20 osd.6 22 _dispatch 0x5557e75e3a40 command(tid 14: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:00.644+0000 7f2b947b0700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] --> 172.21.15.131:0/890522420 -- command_reply(tid 14: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x5557e7563500 con 0x5557e7450400 2019-12-07T19:34:00.648+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 388190 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 71 ==== osd_map(22..23 src has 1..23) v4 ==== 1258+0+0 (crc 0 0 0) 0x5557e9c13440 con 0x5557e79cbc00 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 osd.6 22 OSD::ms_dispatch: osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 osd.6 22 do_waiters -- start 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 osd.6 22 do_waiters -- finish 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 osd.6 22 _dispatch 0x5557e9c13440 osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.651+0000 7f2b834f8700 3 osd.6 22 handle_osd_map epochs [22,23], i have 22, src has [1,23] 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 osd.6 22 handle_osd_map got inc map for epoch 23 2019-12-07T19:34:00.651+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3760e826:::osdmap.22:0# 0x0~0 2019-12-07T19:34:00.651+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 72 ==== MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 141592+0+0 (crc 0 0 0) 0x5557e9d81800 con 0x5557e79cbc00 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_read will do buffered read 2019-12-07T19:34:00.651+0000 7f2b957b2700 15 osd.6 22 enqueue_op 0x5557e75c4580 prio 127 cost 0 latency 0.000024 epoch 23 MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.651+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9d369a0 blob([0x67000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9d367e0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _read_cache blob Blob(0x5557e9d92000 blob([!~1000,0x68000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x5557e9b56c40 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) read meta #-1:3760e826:::osdmap.22:0# 0x0~140c = 5132 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 osd.6 22 add_map_bl 22 5132 bytes 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 osd.6 22 got_full_map 23, nothing requested 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 osd.6 22 handle_osd_map pg_num_history pg_num_history(e23 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps purged_snaps {23={}} 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 osd.6 22 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.6 429b1431-873f-4985-95c1-51e7035621d2 e23 [1,23] lci=[13,23]) 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6280960 meta 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce500 = 0x5557ea18b500 seq 14 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:c57b972f:::inc_osdmap.23:0# key 0x7f7fffffffffffffffc57b97'/!inc_osdmap.23!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1131 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~269 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557ea18b500 1 blobs 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x1000/1000,1000,0 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x69000~1000/1000,1000,0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x69000~1000] 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3110 blob([0x69000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3180 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e9ba3110) get_ref 0x0~269 Blob(0x5557e9ba3110 blob([0x69000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba3180 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e9ba3110) get_ref init 0x1000, 1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x5557e9ba3110 blob([0x69000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x5557e9ba3180 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e9ba3198 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x269 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e9ff9508) dirty_range mark inline shard dirty 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 = 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) get_onode oid #-1:3de0e826:::osdmap.23:0# key 0x7f7fffffffffffffff3de0e8'&!osdmap.23!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(meta 0x5557e6280960) r -2 v.len 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3de0e826:::osdmap.23:0# 0x0~1444 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1132 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:3de0e826:::osdmap.23:0# 0x0~1444 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x1000~444 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _pad_zeros pad 0x0 + 0xbbc on front/back, now 0x0~1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557ea18b500 2 blobs 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate 0x2000/1000,2000,0 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x6a000~2000/1000,2000,0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x6a000~2000] 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba2e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba2e70 blob([0x6a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2ee0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e9ba2e70) get_ref 0x0~1000 Blob(0x5557e9ba2e70 blob([0x6a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba2ee0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e9ba2e70) get_ref init 0x1000, 1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x5557e9ba2e70 blob([0x6a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9ba2ee0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e9ba2ef8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba2f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7669ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba2f50 blob([!~1000,0x6b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7669ea0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e9ba2f50) get_ref 0x1000~444 Blob(0x5557e9ba2f50 blob([!~1000,0x6b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x5557e7669ea0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e9ba2f50) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1000~444: 0x1000~444 Blob(0x5557e9ba2f50 blob([!~1000,0x6b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,444]) SharedBlob(0x5557e7669ea0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e7669eb8 in 0x5557e6269340) _discard 0x1000~1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x1444 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e9ff9288) dirty_range mark inline shard dirty 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:3de0e826:::osdmap.23:0# 0x0~1444 = 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~26 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e7a91cf8 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) get_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e7a91c70) put_ref 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557ea18b500 0 blobs 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x5557e7a91c70 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x5557e7a91ce0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) dirty_range mark inline shard dirty 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small 0x0~24e 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small considering Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.BufferSpace(0x5557e76cc718 in 0x5557e6269340) _discard 0x0~1000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small deferred write 0x0~1000 of mutable Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) get_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.blob(0x5557e76cc5b0) put_ref 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557ea18b500 0 blobs 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x5557e76cc5b0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x5557e76cc700 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) dirty_range mark inline shard dirty 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea18b500 cost 23286 (4 ios * 4000 + 7286 bytes) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea18b500 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e9ff9180,0x5557e9ff9400 shared_blobs 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e62a1c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e70c7288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e9ff9288) update #-1:3de0e826:::osdmap.23:0# 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e9ff9288) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:3de0e826:::osdmap.23:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e9ff9508) update #-1:c57b972f:::inc_osdmap.23:0# 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore.extentmap(0x5557e9ff9508) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode #-1:c57b972f:::inc_osdmap.23:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea18b500 allocated 0x[69000~3000] released 0x[] 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 freelist allocate 0x69000~3000 2019-12-07T19:34:00.651+0000 7f2b834f8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18b500 prepare 2019-12-07T19:34:00.651+0000 7f2b834f8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557ea18b500 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {23=} 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) pg 0x5557e7954000 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 osd.6 22 dequeue_op 0x5557e75c4580 prio 127 cost 0 latency 0.000602 MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 osd.6 22 maybe_share_map con 0x5557e79cbc00 v2:172.21.15.109:6810/36938 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=36, reqid=client.4540.0:177, at_version=23'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [23'8 (21'5) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:177 2019-12-07T19:34:00.649423+0000 0 ObjectCleanRegions clean_offsets: [0~442368,999424~18446744073708552191], clean_omap: 1, new_object: 0] 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 23'8 (21'5) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:177 2019-12-07T19:34:00.649423+0000 0 ObjectCleanRegions clean_offsets: [0~442368,999424~18446744073708552191], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e9b11b00 seq 16 2019-12-07T19:34:00.651+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 2 keys 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 2 keys = 0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 0x1b000~22000 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:cd22b62d:::smithi13140796-27:head# 0x1b000~22000 - have 0x1b000 (110592) bytes fadvise_flags 0x20 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x1b000~22000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9b11b00 17 blobs 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate 0x22000/1000,22000,0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x6c000~3000/1000,22000,0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x1a9000~f000/1000,22000,0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x235000~10000/1000,22000,0 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x6c000~3000,0x1a9000~f000,0x235000~10000] 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb0c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d11650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb0c40 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d11650 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0c40) get_ref 0x0~2000 Blob(0x5557e9cb0c40 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d11650 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0c40) get_ref init 0x2000, 1000 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x5557e9cb0c40 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d11650 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d11668 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36e00 blob([0x6e000~1000,0x1a9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) 2019-12-07T19:34:00.651+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9c36e00) get_ref 0x0~2000 Blob(0x5557e9c36e00 blob([0x6e000~1000,0x1a9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9c36e00) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x5557e9c36e00 blob([0x6e000~1000,0x1a9000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0cb0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0cc8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb12d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb12d0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0b60 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb12d0) get_ref 0x0~2000 Blob(0x5557e9cb12d0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0b60 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb12d0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x5557e9cb12d0 blob([0x1aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0b60 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0b78 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb0bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb0bd0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1260 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0bd0) get_ref 0x0~2000 Blob(0x5557e9cb0bd0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb1260 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x5557e9cb0bd0 blob([0x1ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb1260 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb1278 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9b7b9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9b7b9d0 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0a80 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b7b9d0) get_ref 0x0~2000 Blob(0x5557e9b7b9d0 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0a80 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9b7b9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x5557e9b7b9d0 blob([0x1ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0a80 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0a98 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9cb0af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb11f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9cb0af0 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb11f0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0af0) get_ref 0x0~2000 Blob(0x5557e9cb0af0 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb11f0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9cb0af0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x5557e9cb0af0 blob([0x1b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb11f0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb1208 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2bf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2bf80 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9730 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9c2bf80) get_ref 0x0~2000 Blob(0x5557e9c2bf80 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e74d9730 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9c2bf80) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x5557e9c2bf80 blob([0x1b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74d9730 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e74d9748 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9be2b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9be2b60 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0e00 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9be2b60) get_ref 0x0~2000 Blob(0x5557e9be2b60 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0e00 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9be2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x5557e9be2b60 blob([0x1b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0e00 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0e18 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d10d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d10d20 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93180 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d10d20) get_ref 0x0~2000 Blob(0x5557e9d10d20 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d93180 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d10d20) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x5557e9d10d20 blob([0x1b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d93180 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9d93198 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d93260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d93260 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0d90 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d93260) get_ref 0x0~2000 Blob(0x5557e9d93260 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0d90 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9d93260) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x5557e9d93260 blob([0x235000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0d90 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0da8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e76207e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e76207e0 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70690 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e76207e0) get_ref 0x0~2000 Blob(0x5557e76207e0 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70690 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e76207e0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x5557e76207e0 blob([0x237000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70690 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9b706a8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c2ba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c2ba40 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0d20 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9c2ba40) get_ref 0x0~2000 Blob(0x5557e9c2ba40 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9cb0d20 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9c2ba40) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x5557e9c2ba40 blob([0x239000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9cb0d20 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9cb0d38 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea248e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea248e00 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b7a0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea248e00) get_ref 0x0~2000 Blob(0x5557ea248e00 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b7a0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557ea248e00) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x5557ea248e00 blob([0x23b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b7a0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9e5b7b8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b8f0 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b880 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9e5b8f0) get_ref 0x0~2000 Blob(0x5557e9e5b8f0 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b880 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9e5b8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x5557e9e5b8f0 blob([0x23d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b880 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9e5b898 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b9d0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b960 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9e5b9d0) get_ref 0x0~2000 Blob(0x5557e9e5b9d0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b960 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9e5b9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x5557e9e5b9d0 blob([0x23f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b960 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9e5b978 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5bab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ba40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5bab0 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ba40 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9e5bab0) get_ref 0x0~2000 Blob(0x5557e9e5bab0 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5ba40 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9e5bab0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x5557e9e5bab0 blob([0x241000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5ba40 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9e5ba58 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b6c0 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bea0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9e5b6c0) get_ref 0x0~2000 Blob(0x5557e9e5b6c0 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5bea0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.blob(0x5557e9e5b6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x5557e9e5b6c0 blob([0x243000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5bea0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.BufferSpace(0x5557e9e5beb8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x3d000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e76cad88) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 0x1b000~22000 = 0 2019-12-07T19:34:00.652+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:00.652+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000023.00000000000000000008' <- 0000000023.00000000000000000008 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11b00 cost 218050 (19 ios * 4000 + 142050 bytes) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11b00 onodes 0x5557e76cac80 shared_blobs 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e76cad88) update 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore.extentmap(0x5557e76cad88) update inline shard 595 bytes from 31 extents 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:cd22b62d:::smithi13140796-27:head# is 1086 (485 bytes onode + 2 bytes spanning blobs + 599 bytes inline extents) 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11b00 allocated 0x[6c000~3000,1a9000~f000,235000~10000] released 0x[] 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 freelist allocate 0x6c000~3000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 freelist allocate 0x1a9000~f000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 freelist allocate 0x235000~10000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 prepare 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9b11b00 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 23'8 2019-12-07T19:34:00.652+0000 7f2b74cdb700 10 osd.6 22 dequeue_op 0x5557e75c4580 finish 2019-12-07T19:34:00.652+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.652+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18b500 aio_wait 2019-12-07T19:34:00.652+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea18b500 2019-12-07T19:34:00.652+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18b500 io_done 2019-12-07T19:34:00.652+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 aio_wait 2019-12-07T19:34:00.652+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11b00 2019-12-07T19:34:00.652+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 io_done 2019-12-07T19:34:00.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 1 stable 0 2019-12-07T19:34:00.652+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.652+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e62a1b80 had 1 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70c7180 had 1 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9ff9180 had 1 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9ff9400 had 1 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e70ce500 had 1 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 11887 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cac80 had 1 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 11934 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 11934 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x7c54d~2e9e to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x7c54d 0x7c54d~2e9e to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4c54d 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x54d 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x3eb and padding block with 0xc15 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x7f3eb 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 1 in 0.000235036s (0.000005473s flush + 0.000229563s kv commit) 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557ea18b500,0x5557e9b11b00> 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <0x5557e9b5adc0> 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18b500 kv_submitted 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea18b500 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _deferred_queue txc 0x5557ea18b500 osr 0x5557e76ce500 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557ea297040) prepare_write seq 12 0x1b000~1000 crc 344afd6c 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.DeferredBatch(0x5557ea297040) prepare_write seq 12 0x2000~1000 crc fddfad23 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 kv_submitted 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11b00 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11b00 finishing 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11b00 onodes 0x5557e76cac80 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e74d9748 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221b90 space 0x5557e74d9748 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b706a8 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221dd0 space 0x5557e9b706a8 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0a98 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221a70 space 0x5557e9cb0a98 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0b78 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea07f050 space 0x5557e9cb0b78 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0cc8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e44750 space 0x5557e9cb0cc8 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0d38 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221e60 space 0x5557e9cb0d38 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0da8 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221d40 space 0x5557e9cb0da8 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb0e18 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221c20 space 0x5557e9cb0e18 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1208 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221b00 space 0x5557e9cb1208 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9cb1278 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e450e0 space 0x5557e9cb1278 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d11668 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9e45170 space 0x5557e9d11668 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d93198 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221cb0 space 0x5557e9d93198 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b7b8 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea221ef0 space 0x5557e9e5b7b8 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b898 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c14b40 space 0x5557e9e5b898 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b978 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c14bd0 space 0x5557e9e5b978 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5ba58 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c14c60 space 0x5557e9e5ba58 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5beb8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c14cf0 space 0x5557e9e5beb8 0x0~2000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11b00 done 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11b00 [] 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10000 deferred_cleanup 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10000 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7606f00,0x5557e7607180,0x5557e7607400,0x5557e7607680 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e6f54398 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab830 space 0x5557e6f54398 0x0~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56088 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa870 space 0x5557e9b56088 0x0~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56168 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab200 space 0x5557e9b56168 0x1000~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56328 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa900 space 0x5557e9b56328 0x0~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56478 in 0x5557e6269340) _finish_write discard buffer(0x5557e9b58000 space 0x5557e9b56478 0x0~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56558 in 0x5557e6269340) _finish_write discard buffer(0x5557e9b58090 space 0x5557e9b56558 0x1000~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10000 done 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93500 deferred_cleanup 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10000 [] 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93500 deferred_cleanup 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93500 onodes 0x5557e62a1b80,0x5557e70c7180,0x5557e7557680,0x5557e7557900 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b56c58 in 0x5557e6269340) _finish_write discard buffer(0x5557e79aa7e0 space 0x5557e9b56c58 0x1000~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9be2328 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab4d0 space 0x5557e9be2328 0x0~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d367f8 in 0x5557e6269340) _finish_write discard buffer(0x5557e79ab560 space 0x5557e9d367f8 0x0~1000 writing nocache) 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93500 done 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea18b500 deferred_queued 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93500 [] 2019-12-07T19:34:00.653+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.653+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6 22 _committed_osd_maps 23..23 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6 22 advance to epoch 23 (<= last 23 <= newest_map 23) 2019-12-07T19:34:00.653+0000 7f2b85cfd700 7 osd.6 23 consume_map version 23 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.8s5 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.1s2 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.9s5 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.2s3 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.as1 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.bs1 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.3s5 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.cs0 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.5s1 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.6s1 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.es5 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.7s1 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 identify_splits_and_merges 1.fs5 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6 23 prune_pg_created 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6:0.consume_map 23 (was 22) 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:0.consume_map consume_map 1.8s5 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:0.consume_map consume_map 1.8s5 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6:1.consume_map 23 (was 22) 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.1s2 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.1s2 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:1._wake_pg_slot _wake_pg_slot 1.1s2 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.9s5 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:1.consume_map consume_map 1.9s5 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:1._wake_pg_slot _wake_pg_slot 1.9s5 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s)) prio 255 cost 10 e23) pg 0x5557e76ba000 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6:2.consume_map 23 (was 22) 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.2s3 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.as1 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:2.consume_map consume_map 1.as1 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:2._wake_pg_slot _wake_pg_slot 1.as1 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6:3.consume_map 23 (was 22) 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.bs1 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.bs1 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.3s5 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:3.consume_map consume_map 1.3s5 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6:4.consume_map 23 (was 22) 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:4.consume_map consume_map 1.cs0 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6:5.consume_map 23 (was 22) 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:5.consume_map consume_map 1.5s1 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6:6.consume_map 23 (was 22) 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.6s1 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6:0.update_pg_epoch 1.8s5 22 -> 23 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.6s1 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 22 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] handle_advance_map [1,4,5,2,3,6]/[1,4,5,2,3,6] -- 1/1 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:6._wake_pg_slot _wake_pg_slot 1.6s1 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.es5 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:6.consume_map consume_map 1.es5 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:6._wake_pg_slot _wake_pg_slot 1.es5 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f2b85cfd700 10 osd.6:7.consume_map 23 (was 22) 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.7s1 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.7s1 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:7._wake_pg_slot _wake_pg_slot 1.7s1 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.fs5 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:7.consume_map consume_map 1.fs5 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.653+0000 7f2b85cfd700 20 osd.6:7._wake_pg_slot _wake_pg_slot 1.fs5 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.8s5_head options 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) pg 0x5557e766d000 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) pg 0x5557e766a000 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6:2.update_pg_epoch 1.as1 22 -> 23 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6:6.update_pg_epoch 1.es5 22 -> 23 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 22 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s)) prio 255 cost 10 e23) pg 0x5557e7954000 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.8s0( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.078095466s ub 36.078095466s int 16s) 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.as1_head options 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.es5_head options 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6:3.update_pg_epoch 1.3s5 22 -> 23 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] handle_advance_map [4,1,2,5,7,6]/[4,1,2,5,7,6] -- 4/4 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9cc5200 seq 26 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.3s5_head options 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5200 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5200 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 prepare 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5200 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 io_done 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] activate_map: Not dirtying info: last_persisted is 22 while current is 23 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2d20 1.as1_head 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9e00 = 0x5557e7692900 seq 9 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.3s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.064119817s ub 36.064119817s int 16s) 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e34a0 1.es5_head 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.as1_head 1#1:50000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c9a40 = 0x5557ea18c000 seq 13 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692900 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692900 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e7693800 seq 17 2019-12-07T19:34:00.653+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 prepare 2019-12-07T19:34:00.653+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692900 2019-12-07T19:34:00.653+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 io_done 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.653+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.es5_head 5#1:70000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea18c000 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea18c000 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea18c000 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693800 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693800 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c000 prepare 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea18c000 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c000 io_done 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 prepare 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693800 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 io_done 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s)) prio 255 cost 10 e23) pg 0x5557e769c000 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:34:00.653+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 23/18 ECSubWriteReply(tid=36, last_complete=23'8, committed=1, applied=1)) v2 -- 0x5557e9b37a00 con 0x5557e79cbc00 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cf00 had 1 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2259 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2259 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x7f3eb~8d3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6:6.update_pg_epoch 1.6s1 22 -> 23 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x7f3eb 0x7f3eb~8d3 to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4f3eb 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x3eb 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xcbe and padding block with 0x342 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 22 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map [3,6,7,1,2,5]/[3,6,7,1,2,5] -- 3/3 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) pg 0x5557e7957000 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.6s1_head options 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6:3.update_pg_epoch 1.bs1 22 -> 23 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 22 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 73 ==== MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 96536+0+0 (crc 0 0 0) 0x5557e9d82800 con 0x5557e79cbc00 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f2b957b2700 15 osd.6 23 enqueue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000027 epoch 23 MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.653+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.bs1_head options 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.6s0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075844307s ub 36.075844307s int 16s) 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557ea18c300 seq 24 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) pg 0x5557e767c000 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea18c300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea18c300 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea18c300 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b41e0 1.bs1_head 2019-12-07T19:34:00.653+0000 7f2b75cdd700 10 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e79ed2c0 = 0x5557e7693200 seq 23 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6:1.update_pg_epoch 1.9s5 22 -> 23 2019-12-07T19:34:00.653+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c300 prepare 2019-12-07T19:34:00.653+0000 7f2b75cdd700 10 osd.6 pg_epoch: 22 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea18c300 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c300 io_done 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:00.653+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7693200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7693200 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7693200 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 prepare 2019-12-07T19:34:00.653+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.9s5_head options 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x7fcbe 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7693200 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 io_done 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.653+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 2 cleaned 0 in 0.000216846s (0.000000094s flush + 0.000216752s kv commit) 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) pg 0x5557e7954000 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 834 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1129 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7f2b78ce3700 10 osd.6 23 dequeue_op 0x5557e75c58c0 prio 127 cost 0 latency 0.000149 MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795cc80 had 1 2019-12-07T19:34:00.653+0000 7f2b75cdd700 20 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2247 < min_flush_size 524288 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=37, reqid=client.4540.0:178, at_version=23'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3365 < min_flush_size 524288 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4483 < min_flush_size 524288 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'8 (0'0,23'8] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] append_log log((0'0,23'8], crt=21'7) [23'9 (23'8) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:178 2019-12-07T19:34:00.651599+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1376256~18446744073708175359], clean_omap: 1, new_object: 0] 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c280 had 1 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4506 < min_flush_size 524288 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4506 < min_flush_size 524288 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e2b40 1.9s5_head 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678280 = 0x5557e9d5a900 seq 13 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=23'8 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] add_log_entry 23'9 (23'8) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:178 2019-12-07T19:34:00.651599+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1376256~18446744073708175359], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x7fcbe~119a to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.654+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=23'8 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log approx pg log length = 9 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=23'8 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:00.654+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.9s5_head 5#1:90000000::::head# = 0 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5a900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5a900 onodes shared_blobs 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5a900 allocated 0x[] released 0x[] 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x7fcbe 0x7fcbe~119a to file(ino 16 size 0x80000 mtime 2019-12-07T19:33:58.822179+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _allocate len 0xe58 from 0 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _allocate len 0xe58 from 1 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate 0x10000/10000,10000,acd1b0000 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 fbmap_alloc 0x5557e6283100 allocate extent: 0xacd1b0000~10000/10000,10000,acd1b0000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a900 prepare 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush_range extending WAL size to 0x90000 to include allocated 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5a900 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range dirty_seq = 30 (was clean) 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a900 io_done 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x4fcbe 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xcbe 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e9d5af00 seq 18 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe58 and padding block with 0x1a8 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range zeroing WAL log up to 0x90000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 2 keys 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) pg 0x5557e769f000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 2 keys = 0 2019-12-07T19:34:00.654+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:00.654+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 0x3d000~17000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:cd22b62d:::smithi13140796-27:head# 0x3d000~17000 - have 0x3d000 (249856) bytes fadvise_flags 0x20 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x3d000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6:1.update_pg_epoch 1.1s2 22 -> 23 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 22 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9d5af00 12 blobs 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] state: Started advmap 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.1s2_head options 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] hit_set_clear 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] agent_stop 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] activate_map 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate 0x17000/1000,17000,0 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x245000~12000/1000,17000,0 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2c9000~5000/1000,17000,0 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x245000~12000,0x2c9000~5000] 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d10f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9d10ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d10f50 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d10ee0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d10f50) get_ref 0x0~2000 Blob(0x5557e9d10f50 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9d10ee0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d10f50) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x5557e9d10f50 blob([0x245000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9d10ee0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9d10ef8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9d11570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea081ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9d11570 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8000 1.1s2_head 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d11570) get_ref 0x0~2000 Blob(0x5557e9d11570 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea081ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9d11570) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76783c0 = 0x5557e9d5b200 seq 11 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x5557e9d11570 blob([0x247000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea081ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557ea081ac8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5aee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7453f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5aee0 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453f10 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._biginfo' <- _biginfo 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9e5aee0) get_ref 0x0~2000 Blob(0x5557e9e5aee0 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7453f10 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9e5aee0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._epoch' <- _epoch 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x5557e9e5aee0 blob([0x249000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7453f10 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000435'._info' <- _info 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e7453f28 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.1s2_head 2#1:80000000::::head# = 0 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5b200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f08b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5b200 onodes shared_blobs 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5b200 allocated 0x[] released 0x[] 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f08b60 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08bd0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9f08b60) get_ref 0x0~2000 Blob(0x5557e9f08b60 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08bd0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9f08b60) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x5557e9f08b60 blob([0x24b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08bd0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b200 prepare 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9f08be8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5b200 2019-12-07T19:34:00.654+0000 7f2b75cdd700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b200 io_done 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9f08c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9f08c40 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08cb0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9f08c40) get_ref 0x0~2000 Blob(0x5557e9f08c40 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f08cb0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9f08c40) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x5557e9f08c40 blob([0x24d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f08cb0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9f08cc8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1d50 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ce0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1d50) get_ref 0x0~2000 Blob(0x5557e9ec1d50 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ce0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 10 osd.6 23 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1d50) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x5557e9ec1d50 blob([0x24f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1ce0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x80e58 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9ec1cf8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.8s5 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1e30 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1dc0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.1s2 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1e30) get_ref 0x0~2000 Blob(0x5557e9ec1e30 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1dc0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1e30) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x5557e9ec1e30 blob([0x251000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1dc0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.9s5 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9ec1dd8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(1) _enqueue OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.2s3 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1ea0 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1f10 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.as1 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1ea0) get_ref 0x0~2000 Blob(0x5557e9ec1ea0 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1f10 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x5557e9ec1ea0 blob([0x253000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1f10 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.bs1 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9ec1f28 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.3s5 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1f80 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.cs0 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1f80) get_ref 0x0~2000 Blob(0x5557e9ec1f80 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1f80) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x5557e9ec1f80 blob([0x255000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1ab0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9ec1ac8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.5s1 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1a40 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1b90 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.6s1 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1a40) get_ref 0x0~2000 Blob(0x5557e9ec1a40 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1b90 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1a40) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.es5 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x5557e9ec1a40 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1b90 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9ec1ba8 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.7s1 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 15 osd.6 23 enqueue_peering_evt 1.fs5 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1b20 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1c70 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _fsync file metadata was dirty (30) on file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]), flushing log 2019-12-07T19:34:00.654+0000 7f2b85cfd700 20 osd.6 op_wq(7) _enqueue OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1b20) get_ref 0x0~2000 Blob(0x5557e9ec1b20 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec1c70 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1b20) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b85cfd700 7 osd.6 23 activate_map version 23 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x51000~2000: 0x0~2000 Blob(0x5557e9ec1b20 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ec1c70 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9ec1c88 in 0x5557e62696c0) _discard 0x0~2000 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush_and_sync_log txn(seq 30 len 0x49 crc 0x809d4f52) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ec1c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec19d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush 0x5557e63f5b80 0x1d000~1000 to file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e63f5b80 pos 0x1d000 0x1d000~1000 to file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ec1c00 blob([!~1000,0x2cd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec19d0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 1 size 0x1e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1d000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1c00) get_ref 0x1000~1000 Blob(0x5557e9ec1c00 blob([!~1000,0x2cd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ec19d0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.blob(0x5557e9ec1c00) get_ref init 0x2000, 1000 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x53000~1000: 0x1000~1000 Blob(0x5557e9ec1c00 blob([!~1000,0x2cd000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x5557e9ec19d0 sbid 0x0)) 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.BufferSpace(0x5557e9ec19e8 in 0x5557e62696c0) _discard 0x1000~1000 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e6e89000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x54000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cad88) dirty_range mark inline shard dirty 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 0x3d000~17000 = 0 2019-12-07T19:34:00.654+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] handle_advance_map: 23 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6:4.update_pg_epoch 1.cs0 22 -> 23 2019-12-07T19:34:00.654+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000023.00000000000000000009' <- 0000000023.00000000000000000009 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 22 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] handle_advance_map [6,5,4,1,2,3]/[6,5,4,1,2,3] -- 6/6 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e63f5b80 pos now 0x1e000 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean] state: Active advmap 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5af00 cost 148186 (13 ios * 4000 + 96186 bytes) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5af00 onodes 0x5557e76cac80 shared_blobs 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cad88) update 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] on_active_advmap new removed_snaps [2~1], snap_trimq now [2~1] 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cad88) update inline shard 836 bytes from 43 extents 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] state: Started advmap 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:cd22b62d:::smithi13140796-27:head# is 1327 (485 bytes onode + 2 bytes spanning blobs + 840 bytes inline extents) 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5af00 allocated 0x[245000~12000,2c9000~5000] released 0x[] 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 freelist allocate 0x245000~12000 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 freelist allocate 0x2c9000~5000 2019-12-07T19:34:00.654+0000 7f2b744da700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.cs0_head options 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] plpg_on_pool_change 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 prepare 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9d5af00 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] hit_set_clear 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] agent_stop 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] handle_activate_map: 23 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] activate_map 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] state: Active: handling ActMap 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] Active: kicking snap trim 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] exit NotTrimming 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] enter Trimming 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] enter Trimming/WaitReservation 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.654+0000 7f2b744da700 15 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] publish_stats_to_osd 21:32 2019-12-07T19:34:00.654+0000 7f2b744da700 7 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] state: handle ActMap primary 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.654+0000 7f2b744da700 15 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] publish_stats_to_osd 22:33 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=23'8 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] op_applied version 23'9 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.654+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 23 dequeue_op 0x5557e75c58c0 finish 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b744da700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b744da700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e7957000 2019-12-07T19:34:00.654+0000 7f2b7ace7700 20 osd.6 23 agent_entry empty queue 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e7954000 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b78ce3700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] SnapTrimmer state: NotTrimming: trimming 2 2019-12-07T19:34:00.654+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim_wait trimq=[2~1]] exit Trimming/WaitReservation 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) pg 0x5557e764f000 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 15 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] publish_stats_to_osd 23:34 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 10 osd.6 23 queueing pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean trimq=[2~1]] for snaptrim 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6:7.update_pg_epoch 1.fs5 22 -> 23 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 20 osd.6 op_wq(4) _enqueue OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=23) prio 5 cost 1048576 e23) 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.1s2 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e764c000 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.1s2 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e769f000 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=23) prio 5 cost 1048576 e23) queued 2019-12-07T19:34:00.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b79ce5700 10 osd.6 pg_epoch: 23 pg[1.1s2( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=2 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.fs5_head options 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.654+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e76ba000 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6:5.update_pg_epoch 1.5s1 22 -> 23 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process 1.9s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b764de700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process OpSchedulerItem(1.9s5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e767c000 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 osd.6 pg_epoch: 22 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [2,6,3,4,1,5]/[2,6,3,4,1,5] -- 2/2 2019-12-07T19:34:00.654+0000 7f2b764de700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b79ce5700 10 osd.6 pg_epoch: 23 pg[1.9s5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.654+0000 7f2b79ce5700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.654+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e63f5b80 done in 0.000273 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.654+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e769c000 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.654+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.5s1_head options 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.654+0000 7f2b774e0700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.654+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b7d4ec700 15 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim trimq=[2~1]] publish_stats_to_osd 23:35 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process 1.es5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.es5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e766a000 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 op_wq(4) _process 1.cs0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 op_wq(4) _process OpSchedulerItem(1.cs0 PGSnapTrim(pgid=1.cs0epoch_queued=23) prio 5 cost 1048576 e23) pg 0x5557e6e89000 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.654+0000 7f2b774e0700 10 osd.6 pg_epoch: 23 pg[1.es5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b774e0700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim trimq=[2~1]] snap_trimmer posting 2019-12-07T19:34:00.654+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc5200,0x5557e7692900> 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 kv_submitted 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5200 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim trimq=[2~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 2 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e32c0 1.fs5_head 2019-12-07T19:34:00.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5200 finishing 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5200 onodes 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c83c0 = 0x5557e7a92f00 seq 17 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5200 done 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:34:00.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5200 [] 2019-12-07T19:34:00.654+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.654+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b77ce1700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 kv_submitted 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:00.654+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692900 2019-12-07T19:34:00.654+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 bluestore.onode(0x5557e70ce280).flush flush done 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) get_omap_iterator has_omap = 0 2019-12-07T19:34:00.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692900 finishing 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692900 onodes 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692900 done 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9e00 q now empty 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:00.654+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692900 [] 2019-12-07T19:34:00.654+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:00.654+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.fs5_head 5#1:f0000000::::head# = 0 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92f00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92f00 onodes shared_blobs 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92f00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 prepare 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92f00 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 io_done 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e767f000 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000002_.0_0000000000000001.C) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) pg 0x5557e76bd000 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim trimq=[2~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim] SnapTrimmer state: adding snap 2 to purged_snaps 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6:2.update_pg_epoch 1.2s3 22 -> 23 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6:7.update_pg_epoch 1.7s1 22 -> 23 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 22 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 osd.6 pg_epoch: 22 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] state: Started advmap 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.654+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.7s1_head options 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] state: Started advmap 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim ps=[2~1]] SnapTrimmer state: purged_snaps now [2~1], snap_trimq now [] 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b54a0 1.cs0_head 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] hit_set_clear 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76ce000 = 0x5557e7692600 seq 23 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] agent_stop 2019-12-07T19:34:00.654+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) set_collection_opts 1.2s3_head options 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.654+0000 7f2b784e2700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] hit_set_clear 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.cs0_head 0#1:30000000::::head# = 0 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] agent_stop 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7692600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7692600 onodes shared_blobs 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7692600 allocated 0x[] released 0x[] 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 prepare 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a92c0 1.7s1_head 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7692600 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 io_done 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678dc0 = 0x5557e7a93b00 seq 22 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map 2019-12-07T19:34:00.654+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b72cd7700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._biginfo' <- _biginfo 2019-12-07T19:34:00.654+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.as1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._epoch' <- _epoch 2019-12-07T19:34:00.654+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.as1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e766d000 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000437'._info' <- _info 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim ps=[2~1]] share_pg_info 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.7s1_head 1#1:e0000000::::head# = 0 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a93b00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a93b00 onodes shared_blobs 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a93b00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.654+0000 7f2b794e4700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 prepare 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a93b00 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 io_done 2019-12-07T19:34:00.654+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.as1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.cs3 1.cs3 1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x5557e9b13c00 con 0x5557e79cac00 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.7s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.7s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e76bd000 2019-12-07T19:34:00.654+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.7s1( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=1 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process 1.fs5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process OpSchedulerItem(1.fs5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x5557e764f000 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 osd.6 pg_epoch: 23 pg[1.fs5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.654+0000 7f2b72cd7700 10 log is not dirty 2019-12-07T19:34:00.654+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.cs4 1.cs4 1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x5557e9dcae00 con 0x5557e7519800 2019-12-07T19:34:00.654+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.cs5 1.cs5 1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x5557e9dcd100 con 0x5557e79ca800 2019-12-07T19:34:00.654+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.cs2 1.cs2 1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x5557e9dcd800 con 0x5557e79cbc00 2019-12-07T19:34:00.654+0000 7f2b784e2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.cs1 1.cs1 1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x5557ea18e000 con 0x5557e79ca000 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim ps=[2~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean+snaptrim ps=[2~1]] exit Trimming 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:00.654+0000 7f2b784e2700 15 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean ps=[2~1]] publish_stats_to_osd 23:36 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean ps=[2~1]] enter NotTrimming 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean ps=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean ps=[2~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:00.654+0000 7f2b784e2700 10 osd.6 pg_epoch: 23 pg[1.cs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [6,5,4,1,2,3]p6(0) r=0 lpr=18 crt=21'12 lcod 21'11 mlcod 21'11 active+clean ps=[2~1]] snap_trimmer complete 2019-12-07T19:34:00.654+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 22 ==== osd_map(22..23 src has 1..23) v4 ==== 1258+0+0 (crc 0 0 0) 0x5557ea026240 con 0x5557e79cb800 2019-12-07T19:34:00.654+0000 7f2b834f8700 20 osd.6 23 OSD::ms_dispatch: osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.654+0000 7f2b834f8700 10 osd.6 23 do_waiters -- start 2019-12-07T19:34:00.654+0000 7f2b834f8700 10 osd.6 23 do_waiters -- finish 2019-12-07T19:34:00.654+0000 7f2b834f8700 20 osd.6 23 _dispatch 0x5557ea026240 osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.654+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 23 ==== MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 1364+0+0 (crc 0 0 0) 0x5557e9fcb800 con 0x5557e79cb800 2019-12-07T19:34:00.654+0000 7f2b834f8700 3 osd.6 23 handle_osd_map epochs [22,23], i have 23, src has [1,23] 2019-12-07T19:34:00.654+0000 7f2b834f8700 10 osd.6 23 no new maps here, dropping 2019-12-07T19:34:00.654+0000 7f2b967b4700 15 osd.6 23 enqueue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000025 epoch 23 MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:34:00.654+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e23) pg 0x5557e767f000 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 23 dequeue_op 0x5557e9d84160 prio 127 cost 0 latency 0.000080 MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 23 maybe_share_map con 0x5557e79cb800 v2:172.21.15.131:6822/37079 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] _handle_message: MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log log((0'0,21'9], crt=21'9) [23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0] 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] add_log_entry 23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.655+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] append_log approx pg log length = 10 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.655+0000 7f2b794e4700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.655+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 74 ==== MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2294+0+0 (crc 0 0 0) 0x5557e9d80000 con 0x5557e79cbc00 2019-12-07T19:34:00.655+0000 7f2b957b2700 15 osd.6 23 enqueue_op 0x5557e75c5600 prio 127 cost 0 latency 0.000025 epoch 23 MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:34:00.655+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557ea18d800 seq 16 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.2s3_head 0x5557e75b4d20) get_onode oid 3#1:4c9e444c:::smithi13140796-48:head#a key 0x8380000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffe000000000000000a'o' 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.2s3_head 0x5557e75b4d20) r -2 v.len 0 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.655+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# -> 3#1:4c9e444c:::smithi13140796-48:head#a 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e23) pg 0x5557e7954000 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head# -> 3#1:4c9e444c:::smithi13140796-48:head#a = 0 2019-12-07T19:34:00.655+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.0000000023.00000000000000000010' <- 0000000023.00000000000000000010 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 osd.6 23 dequeue_op 0x5557e75c5600 prio 127 cost 0 latency 0.000065 MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.3s5( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=38, reqid=client.4540.0:179, at_version=23'10, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea18d800 cost 5822 (1 ios * 4000 + 1822 bytes) 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea18d800 onodes 0x5557e9c7d680 shared_blobs 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore.extentmap(0x5557e9c7d788) update 3#1:4c9e444c:::smithi13140796-48:head#a 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 3#1:4c9e444c:::smithi13140796-48:head#a is 966 (485 bytes onode + 2 bytes spanning blobs + 479 bytes inline extents) 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea18d800 allocated 0x[] released 0x[] 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'9 (0'0,23'9] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log log((0'0,23'9], crt=21'7) [23'10 (23'9) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:179 2019-12-07T19:34:00.651608+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18d800 prepare 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea18d800 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18d800 io_done 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=23'9 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] add_log_entry 23'10 (23'9) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:179 2019-12-07T19:34:00.651608+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active mbc={}] op_applied version 23'10 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=23'9 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:00.655+0000 7f2b794e4700 10 osd.6 23 dequeue_op 0x5557e9d84160 finish 2019-12-07T19:34:00.655+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=23'9 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.655+0000 7f2b78ce3700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557ea110000 seq 19 2019-12-07T19:34:00.655+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 3 keys 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# 3 keys = 0 2019-12-07T19:34:00.655+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.3s5_head 5#1:cd22b62d:::smithi13140796-27:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:00.655+0000 7f2b78ce3700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.0000000023.00000000000000000010' <- 0000000023.00000000000000000010 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea110000 cost 5944 (1 ios * 4000 + 1944 bytes) 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea110000 onodes 0x5557e76cac80 shared_blobs 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore.extentmap(0x5557e76cad88) update 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:cd22b62d:::smithi13140796-27:head# is 1327 (485 bytes onode + 2 bytes spanning blobs + 840 bytes inline extents) 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea110000 allocated 0x[] released 0x[] 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea110000 prepare 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea110000 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea110000 blocked by 0x5557e9d5af00 aio_wait 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 lua=23'9 crt=21'7 lcod 23'8 active mbc={} ps=[2~1]] op_applied version 23'10 2019-12-07T19:34:00.655+0000 7f2b78ce3700 10 osd.6 23 dequeue_op 0x5557e75c5600 finish 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluefs _flush_and_sync_log log_seq_stable 30 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 4 cleaned 0 in 0.001838434s (0.000000096s flush + 0.001838338s kv commit) 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557ea18c000,0x5557e7693800,0x5557ea18c300,0x5557e7693200> 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c000 kv_submitted 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea18c000 2019-12-07T19:34:00.655+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 24 ==== pg_info2(1.2s3 1.2s3 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e7699c00 con 0x5557e79cb800 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c000 finishing 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557ea18c000 onodes 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea18c000 done 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c9a40 q now empty 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557ea18c000 [] 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d180 had 1 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f2b967b4700 15 osd.6 23 enqueue_peering_evt 1.2s3 epoch_sent: 23 epoch_requested: 23 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s) 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 kv_submitted 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693800 2019-12-07T19:34:00.655+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795db80 had 1 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693800 finishing 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693800 onodes 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693800 done 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5af00 aio_wait 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693800 [] 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795ca00 had 1 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c300 kv_submitted 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea18c300 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4472 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c300 finishing 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557ea18c300 onodes 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea18c300 done 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557ea18c300 [] 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e7606c80 had 1 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 kv_submitted 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7693200 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7693200 finishing 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7693200 onodes 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7693200 done 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 5590 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e79ed2c0 q now empty 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7693200 [] 2019-12-07T19:34:00.655+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.655+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca280 had 1 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8062 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7d680 had 1 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0a00 had 1 2019-12-07T19:34:00.655+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8085 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 8085 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x80e58~1f95 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.655+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x80e58 0x80e58~1f95 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x50e58 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe58 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.655+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xded and padding block with 0x213 2019-12-07T19:34:00.655+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 aio_wait 2019-12-07T19:34:00.655+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5af00 2019-12-07T19:34:00.655+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 io_done 2019-12-07T19:34:00.655+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea110000 io_done 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s)) prio 255 cost 10 e23) pg 0x5557e767f000 2019-12-07T19:34:00.655+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 lcod 21'9 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 0(0) info: 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.073596236s ub 36.073596236s int 16s) 2019-12-07T19:34:00.655+0000 7f2b754dc700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 lcod 21'9 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.655+0000 7f2b754dc700 10 log is not dirty 2019-12-07T19:34:00.655+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:34:00.655+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557e7a92600 seq 17 2019-12-07T19:34:00.655+0000 7f2b754dc700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:00.655+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:34:00.655+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7a92600 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7a92600 onodes shared_blobs 2019-12-07T19:34:00.655+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7a92600 allocated 0x[] released 0x[] 2019-12-07T19:34:00.656+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 prepare 2019-12-07T19:34:00.656+0000 7f2b754dc700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7a92600 2019-12-07T19:34:00.656+0000 7f2b754dc700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 io_done 2019-12-07T19:34:00.656+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x82ded 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 6 cleaned 0 in 0.000493212s (0.000000081s flush + 0.000493131s kv commit) 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2496 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cac80 had 2 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 2 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 4923 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76cac80 had 1 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6017 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6040 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 6040 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x82ded~1798 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x82ded 0x82ded~1798 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x52ded 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xded 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x585 and padding block with 0xa7b 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5a900,0x5557e9d5b200,0x5557e7a92f00,0x5557e7692600,0x5557e7a93b00,0x5557ea18d800> 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a900 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5a900 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5a900 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5a900 onodes 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5a900 done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678280 q now empty 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5a900 [] 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b200 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5b200 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5b200 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5b200 onodes 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5b200 done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76783c0 q now empty 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5b200 [] 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92f00 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92f00 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92f00 onodes 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92f00 done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c83c0 q now empty 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92f00 [] 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7692600 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7692600 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7692600 onodes 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7692600 done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76ce000 q now empty 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7692600 [] 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a93b00 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a93b00 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a93b00 onodes 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a93b00 done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678dc0 q now empty 2019-12-07T19:34:00.656+0000 7f2b744da700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a93b00 [] 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18d800 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea18d800 2019-12-07T19:34:00.656+0000 7f2b784e2700 20 osd.6 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18d800 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557ea18d800 onodes 0x5557e9c7d680 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea18d800 done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92600 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557ea18d800 [] 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.656+0000 7f2b76cdf700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b75cdd700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b79ce5700 20 osd.6 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x84585 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.656+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.656+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 -- 0x5557e76b0000 con 0x5557e79cb800 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 3 cleaned 0 in 0.000296817s (0.000002849s flush + 0.000293968s kv commit) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.656+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5af00,0x5557ea110000,0x5557e7a92600> 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5af00 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5af00 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5af00 onodes 0x5557e76cac80 2019-12-07T19:34:00.656+0000 7f2b72cd7700 20 osd.6 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7453f28 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c15290 space 0x5557e7453f28 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9d10ef8 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea2219e0 space 0x5557e9d10ef8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec19e8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c157a0 space 0x5557e9ec19e8 0x1000~1000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1ac8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c155f0 space 0x5557e9ec1ac8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1ba8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c15680 space 0x5557e9ec1ba8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1c88 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c15710 space 0x5557e9ec1c88 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1cf8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c15440 space 0x5557e9ec1cf8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1dd8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c154d0 space 0x5557e9ec1dd8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ec1f28 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c15560 space 0x5557e9ec1f28 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f08be8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c15320 space 0x5557e9f08be8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f08cc8 in 0x5557e62696c0) _finish_write discard buffer(0x5557e9c153b0 space 0x5557e9f08cc8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea081ac8 in 0x5557e62696c0) _finish_write discard buffer(0x5557ea07f200 space 0x5557ea081ac8 0x0~2000 writing nocache) 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5af00 done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea110000 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5af00 [] 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea110000 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea110000 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea110000 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557ea110000 onodes 0x5557e76cac80 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea110000 done 2019-12-07T19:34:00.656+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557ea110000 [] 2019-12-07T19:34:00.656+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 23/18 ECSubWriteReply(tid=37, last_complete=23'9, committed=1, applied=1)) v2 -- 0x5557e626fa00 con 0x5557e79cbc00 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 kv_submitted 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7a92600 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7a92600 finishing 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7a92600 onodes 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7a92600 done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7a92600 [] 2019-12-07T19:34:00.656+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.656+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.656+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 23/18 ECSubWriteReply(tid=38, last_complete=23'10, committed=1, applied=1)) v2 -- 0x5557e74f1000 con 0x5557e79cbc00 2019-12-07T19:34:00.656+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.657+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.4 v2:172.21.15.109:6810/36938 75 ==== MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e763e000 con 0x5557e79cbc00 2019-12-07T19:34:00.657+0000 7f2b957b2700 15 osd.6 23 enqueue_op 0x5557e75c5340 prio 127 cost 0 latency 0.000022 epoch 23 MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:00.657+0000 7f2b957b2700 20 osd.6 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.657+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.657+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.657+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.657+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) pg 0x5557e7954000 2019-12-07T19:34:00.657+0000 7f2b74cdb700 10 osd.6 23 dequeue_op 0x5557e75c5340 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 pg pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.657+0000 7f2b74cdb700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.3s5 23/18 ECSubWrite(tid=39, reqid=client.4540.0:179, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:00.657+0000 7f2b74cdb700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=21'7 active mbc={} ps=[2~1]] append_log log((0'0,23'10], crt=21'7) [] 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] rollforward: entry=23'8 (21'5) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:177 2019-12-07T19:34:00.649423+0000 0 ObjectCleanRegions clean_offsets: [0~442368,999424~18446744073708552191], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] rollforward: entry=23'9 (23'8) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:178 2019-12-07T19:34:00.651599+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1376256~18446744073708175359], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] rollforward: entry=23'10 (23'9) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:179 2019-12-07T19:34:00.651608+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 osd.6 pg_epoch: 23 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,1,2,5,7,6]p4(0) r=5 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.658+0000 7f2b74cdb700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4000 1.3s5_head 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e76cfcc0 = 0x5557e9b11800 seq 20 2019-12-07T19:34:00.658+0000 7f2b74cdb700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b11800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b11800 onodes shared_blobs 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b11800 allocated 0x[] released 0x[] 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 prepare 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b11800 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 io_done 2019-12-07T19:34:00.658+0000 7f2b74cdb700 10 osd.6 23 dequeue_op 0x5557e75c5340 finish 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c000 had 1 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x84585~471 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x84585 0x84585~471 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x54585 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x585 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x9f6 and padding block with 0x60a 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x849f6 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.658+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000119336s (0.000000055s flush + 0.000119281s kv commit) 2019-12-07T19:34:00.658+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b11800> 2019-12-07T19:34:00.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 kv_submitted 2019-12-07T19:34:00.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b11800 2019-12-07T19:34:00.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b11800 finishing 2019-12-07T19:34:00.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b11800 onodes 2019-12-07T19:34:00.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b11800 done 2019-12-07T19:34:00.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e76cfcc0 q now empty 2019-12-07T19:34:00.658+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b11800 [] 2019-12-07T19:34:00.658+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.658+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.658+0000 7f2b78ce3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 23/18 ECSubWriteReply(tid=39, last_complete=23'10, committed=1, applied=1)) v2 -- 0x5557e74f1600 con 0x5557e79cbc00 2019-12-07T19:34:00.658+0000 7f2b78ce3700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.658+0000 7f2b74cdb700 20 osd.6 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.659+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 42 ==== pg_info2(1.5s1 1.5s1 1.5s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (crc 0 0 0) 0x5557e9dcae00 con 0x5557e7519800 2019-12-07T19:34:00.659+0000 7f2b95fb3700 15 osd.6 23 enqueue_peering_evt 1.5s1 epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.5s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s) 2019-12-07T19:34:00.659+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.5s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.5s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.5s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s)) prio 255 cost 10 e23) pg 0x5557e764c000 2019-12-07T19:34:00.659+0000 7f2b73cd9700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.5s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.087862669s ub 36.087862669s int 16s) 2019-12-07T19:34:00.659+0000 7f2b73cd9700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.659+0000 7f2b73cd9700 10 log is not dirty 2019-12-07T19:34:00.659+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9cc4000 seq 17 2019-12-07T19:34:00.659+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:00.659+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:34:00.659+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc4000 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc4000 onodes shared_blobs 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc4000 allocated 0x[] released 0x[] 2019-12-07T19:34:00.659+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 prepare 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc4000 2019-12-07T19:34:00.659+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 io_done 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x849f6~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x849f6 0x849f6~475 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x549f6 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x9f6 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe6b and padding block with 0x195 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x84e6b 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.659+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000073259s (0.000000056s flush + 0.000073203s kv commit) 2019-12-07T19:34:00.659+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.659+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.659+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc4000> 2019-12-07T19:34:00.659+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.659+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 kv_submitted 2019-12-07T19:34:00.659+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc4000 2019-12-07T19:34:00.659+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc4000 finishing 2019-12-07T19:34:00.659+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc4000 onodes 2019-12-07T19:34:00.659+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc4000 done 2019-12-07T19:34:00.659+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:34:00.659+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc4000 [] 2019-12-07T19:34:00.659+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.659+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.659+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.659+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.659+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 120 ==== osd_map(22..23 src has 1..23) v4 ==== 1258+0+0 (crc 0 0 0) 0x5557e9c13d40 con 0x5557e79cac00 2019-12-07T19:34:00.659+0000 7f2b834f8700 20 osd.6 23 OSD::ms_dispatch: osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.659+0000 7f2b834f8700 10 osd.6 23 do_waiters -- start 2019-12-07T19:34:00.659+0000 7f2b834f8700 10 osd.6 23 do_waiters -- finish 2019-12-07T19:34:00.659+0000 7f2b834f8700 20 osd.6 23 _dispatch 0x5557e9c13d40 osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.659+0000 7f2b834f8700 3 osd.6 23 handle_osd_map epochs [22,23], i have 23, src has [1,23] 2019-12-07T19:34:00.659+0000 7f2b834f8700 10 osd.6 23 no new maps here, dropping 2019-12-07T19:34:00.659+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 121 ==== MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=93, reqid=client.4540.0:181, at_version=23'14, trim_to=0'0, roll_forward_to=21'13)) v2 ==== 88279+0+0 (crc 0 0 0) 0x5557e9ba0800 con 0x5557e79cac00 2019-12-07T19:34:00.659+0000 7f2b957b2700 15 osd.6 23 enqueue_op 0x5557e75c4840 prio 127 cost 0 latency 0.000022 epoch 23 MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=93, reqid=client.4540.0:181, at_version=23'14, trim_to=0'0, roll_forward_to=21'13)) v2 2019-12-07T19:34:00.659+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=93, reqid=client.4540.0:181, at_version=23'14, trim_to=0'0, roll_forward_to=21'13)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.659+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.659+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=93, reqid=client.4540.0:181, at_version=23'14, trim_to=0'0, roll_forward_to=21'13)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.659+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.659+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=93, reqid=client.4540.0:181, at_version=23'14, trim_to=0'0, roll_forward_to=21'13)) v2) prio 127 cost 0 e23) pg 0x5557e76ba000 2019-12-07T19:34:00.659+0000 7f2b7a4e6700 10 osd.6 23 dequeue_op 0x5557e75c4840 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=93, reqid=client.4540.0:181, at_version=23'14, trim_to=0'0, roll_forward_to=21'13)) v2 pg pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 osd.6 23 maybe_share_map con 0x5557e79cac00 v2:172.21.15.131:6823/37078 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=93, reqid=client.4540.0:181, at_version=23'14, trim_to=0'0, roll_forward_to=21'13)) v2 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 21'13 (0'0,21'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={} ps=[2~1]] append_log log((0'0,21'13], crt=21'13) [23'14 (21'3) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:181 0.000000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0] 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'13 crt=21'13 lcod 21'13 active mbc={} ps=[2~1]] add_log_entry 23'14 (21'3) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:181 0.000000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'13 crt=21'13 lcod 21'13 active mbc={} ps=[2~1]] append_log approx pg log length = 14 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'13 crt=21'13 lcod 21'13 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9cc5800 seq 27 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) get_onode oid 5#1:131421e8:::smithi13140796-7:head#e key 0x858000000000000001131421e821736d'ithi13140796-7!='0xfffffffffffffffe000000000000000e'o' 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.8s5_head 0x5557e76a8960) r -2 v.len 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# -> 5#1:131421e8:::smithi13140796-7:head#e 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# -> 5#1:131421e8:::smithi13140796-7:head#e = 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _assign_nid 1133 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# = 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 3 keys 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 3 keys = 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _set_alloc_hint 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0~15000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write 5#1:131421e8:::smithi13140796-7:head# 0x0~15000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _do_write_big 0x0~15000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write txc 0x5557e9cc5800 11 blobs 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 fbmap_alloc 0x5557e6282e00 allocate 0x15000/1000,15000,0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 fbmap_alloc 0x5557e6282e00 allocate extent: 0x2ce000~15000/1000,15000,0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write prealloc [0x2ce000~15000] 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e7669f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e7669f80 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5110 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e7669f80) get_ref 0x0~2000 Blob(0x5557e7669f80 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9dd5110 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e7669f80) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x5557e7669f80 blob([0x2ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9dd5110 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9dd5128 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9dd5030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e7a289a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9dd5030 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a289a0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd5030) get_ref 0x0~2000 Blob(0x5557e9dd5030 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e7a289a0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9dd5030) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x5557e9dd5030 blob([0x2d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7a289a0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e7a289b8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e75c7650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba30a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e75c7650 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba30a0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e75c7650) get_ref 0x0~2000 Blob(0x5557e75c7650 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9ba30a0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e75c7650) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x5557e75c7650 blob([0x2d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ba30a0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9ba30b8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9ba3030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0a000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9ba3030 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0a000 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9ba3030) get_ref 0x0~2000 Blob(0x5557e9ba3030 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9f0a000 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9ba3030) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x5557e9ba3030 blob([0x2d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0a000 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9f0a018 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea2480e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea248150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea2480e0 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea248150 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea2480e0) get_ref 0x0~2000 Blob(0x5557ea2480e0 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea248150 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea2480e0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x5557ea2480e0 blob([0x2d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea248150 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557ea248168 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea2481c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea248230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea2481c0 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea248230 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea2481c0) get_ref 0x0~2000 Blob(0x5557ea2481c0 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea248230 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea2481c0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x5557ea2481c0 blob([0x2d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea248230 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557ea248248 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea249c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea249c00 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b5e0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea249c00) get_ref 0x0~2000 Blob(0x5557ea249c00 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9e5b5e0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea249c00) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x5557ea249c00 blob([0x2da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5b5e0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9e5b5f8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9e5b110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9e5b110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36af0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9e5b110) get_ref 0x0~2000 Blob(0x5557e9e5b110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9c36af0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9e5b110) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x5557e9e5b110 blob([0x2dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c36af0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9c36b08 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557e9c36b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557e9c36b60 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70070 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c36b60) get_ref 0x0~2000 Blob(0x5557e9c36b60 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557e9b70070 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557e9c36b60) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x5557e9c36b60 blob([0x2de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b70070 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557e9b70088 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e1260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e13b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e1260 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e13b0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea1e1260) get_ref 0x0~2000 Blob(0x5557ea1e1260 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e13b0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea1e1260) get_ref init 0x2000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x5557ea1e1260 blob([0x2e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557ea1e13b0 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557ea1e13c8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write initialize csum setting for new blob Blob(0x5557ea1e1420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e1490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write blob Blob(0x5557ea1e1420 blob([0x2e2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e1490 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea1e1420) get_ref 0x0~1000 Blob(0x5557ea1e1420 blob([0x2e2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x5557ea1e1490 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.blob(0x5557ea1e1420) get_ref init 0x1000, 1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_alloc_write lex 0x14000~1000: 0x0~1000 Blob(0x5557ea1e1420 blob([0x2e2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557ea1e1490 sbid 0x0)) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.BufferSpace(0x5557ea1e14a8 in 0x5557e6269960) _discard 0x0~1000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write extending size to 0x15000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9eb5c88) dirty_range mark inline shard dirty 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0~15000 = 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.0000000023.00000000000000000014' <- 0000000023.00000000000000000014 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9cc5800 cost 136720 (12 ios * 4000 + 88720 bytes) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9cc5800 onodes 0x5557e9bc0f00,0x5557e9eb5b80 shared_blobs 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9bc1008) update 5#1:131421e8:::smithi13140796-7:head#e 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:131421e8:::smithi13140796-7:head#e is 565 (483 bytes onode + 2 bytes spanning blobs + 80 bytes inline extents) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9eb5c88) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore.extentmap(0x5557e9eb5c88) update inline shard 209 bytes from 11 extents 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 699 (484 bytes onode + 2 bytes spanning blobs + 213 bytes inline extents) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9cc5800 allocated 0x[2ce000~15000] released 0x[] 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 freelist allocate 0x2ce000~15000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 prepare 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_aio_submit txc 0x5557e9cc5800 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 lua=21'13 crt=21'13 lcod 21'13 active mbc={} ps=[2~1]] op_applied version 23'14 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 10 osd.6 23 dequeue_op 0x5557e75c4840 finish 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.660+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 aio_wait 2019-12-07T19:34:00.660+0000 7f2b8ef15700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9cc5800 2019-12-07T19:34:00.660+0000 7f2b8ef15700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 io_done 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2954 < min_flush_size 524288 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0f00 had 1 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9eb5b80 had 1 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2977 < min_flush_size 524288 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2977 < min_flush_size 524288 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x84e6b~ba1 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x84e6b 0x84e6b~ba1 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x54e6b 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe6b 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa0c and padding block with 0x5f4 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x85a0c 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.660+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000083200s (0.000003793s flush + 0.000079407s kv commit) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9cc5800> 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.660+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.660+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 kv_submitted 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9cc5800 2019-12-07T19:34:00.660+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9cc5800 finishing 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9cc5800 onodes 0x5557e9bc0f00,0x5557e9eb5b80 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e7a289b8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15b00 space 0x5557e7a289b8 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9b70088 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15ef0 space 0x5557e9b70088 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9ba30b8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15b90 space 0x5557e9ba30b8 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9c36b08 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15e60 space 0x5557e9c36b08 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9dd5128 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15a70 space 0x5557e9dd5128 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9e5b5f8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15dd0 space 0x5557e9e5b5f8 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557e9f0a018 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15c20 space 0x5557e9f0a018 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e13c8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9eb2000 space 0x5557ea1e13c8 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea1e14a8 in 0x5557e6269960) _finish_write discard buffer(0x5557e9eb2090 space 0x5557ea1e14a8 0x0~1000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea248168 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15cb0 space 0x5557ea248168 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore.BufferSpace(0x5557ea248248 in 0x5557e6269960) _finish_write discard buffer(0x5557e9c15d40 space 0x5557ea248248 0x0~2000 writing nocache) 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 23/18 ECSubWriteReply(tid=93, last_complete=23'14, committed=1, applied=1)) v2 -- 0x5557e70e2000 con 0x5557e79cac00 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9cc5800 done 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:34:00.660+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9cc5800 [] 2019-12-07T19:34:00.660+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.660+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.660+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.662+0000 7f2b967b4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.0 v2:172.21.15.131:6822/37079 25 ==== MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e9fcb000 con 0x5557e79cb800 2019-12-07T19:34:00.662+0000 7f2b967b4700 15 osd.6 23 enqueue_op 0x5557e9bf66e0 prio 127 cost 0 latency 0.000019 epoch 23 MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:00.662+0000 7f2b967b4700 20 osd.6 op_wq(2) _enqueue OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process 1.2s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process OpSchedulerItem(1.2s3 PGOpItem(op=MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e23) pg 0x5557e767f000 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 osd.6 23 dequeue_op 0x5557e9bf66e0 prio 127 cost 0 latency 0.000057 MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 pg pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] append_log log((0'0,23'10], crt=21'9) [] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] rollforward: entry=23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 osd.6 pg_epoch: 23 pg[1.2s3( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=3 lpr=18 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.662+0000 7f2b794e4700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75b4d20 1.2s3_head 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678000 = 0x5557ea18db00 seq 18 2019-12-07T19:34:00.662+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head#a onode 0x5557e9c7d680 txc 0x5557ea18db00 2019-12-07T19:34:00.662+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head#a 0x0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff0b60) put_ref 0x0~2000 Blob(0x5557e9ff0b60 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3570 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff33b0) put_ref 0x0~2000 Blob(0x5557e9ff33b0 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3340 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff3260) put_ref 0x0~2000 Blob(0x5557e9ff3260 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff31f0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff3180) put_ref 0x0~2000 Blob(0x5557e9ff3180 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3110 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff30a0) put_ref 0x0~2000 Blob(0x5557e9ff30a0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff3030 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2fc0) put_ref 0x0~2000 Blob(0x5557e9ff2fc0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2f50 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2ee0) put_ref 0x0~2000 Blob(0x5557e9ff2ee0 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2e70 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2e00) put_ref 0x0~2000 Blob(0x5557e9ff2e00 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2d90 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2d20) put_ref 0x0~2000 Blob(0x5557e9ff2d20 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff2cb0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2c40) put_ref 0x0~2000 Blob(0x5557e9ff2c40 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff03f0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff09a0) put_ref 0x0~2000 Blob(0x5557e9ff09a0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0af0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff0930) put_ref 0x0~2000 Blob(0x5557e9ff0930 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7669730 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9d921c0) put_ref 0x0~2000 Blob(0x5557e9d921c0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7a29500 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452230) put_ref 0x0~2000 Blob(0x5557e7452230 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74521c0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74522a0) put_ref 0x0~2000 Blob(0x5557e74522a0 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452310 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74523f0) put_ref 0x0~2000 Blob(0x5557e74523f0 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452150 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74520e0) put_ref 0x0~2000 Blob(0x5557e74520e0 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452070 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452460) put_ref 0x0~2000 Blob(0x5557e7452460 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74524d0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452540) put_ref 0x0~2000 Blob(0x5557e7452540 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e74525b0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452620) put_ref 0x0~2000 Blob(0x5557e7452620 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452690 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff3420) put_ref 0x0~2000 Blob(0x5557e9ff3420 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9ff0d20 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452700) put_ref 0x0~2000 Blob(0x5557e7452700 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452770 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74527e0) put_ref 0x0~2000 Blob(0x5557e74527e0 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452850 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74528c0) put_ref 0x0~2000 Blob(0x5557e74528c0 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7452930 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74529a0) put_ref 0x0~1000 Blob(0x5557e74529a0 blob([0x3ff000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e7452a10 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.extentmap(0x5557e9c7d788) dirty_range mark inline shard dirty 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e9ff0b60 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff3570 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3cf000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff0b60) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3588 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 _rm buffer(0x5557e9f01ef0 space 0x5557e9ff3588 0x0~1000 clean) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3cf000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff3570) put 0x5557e9ff3570 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3588 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e9ff33b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff3340 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3d1000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff33b0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3358 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3d1000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff3340) put 0x5557e9ff3340 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3358 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x5557e9ff3260 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff31f0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3d3000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff3260) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3208 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3d3000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff31f0) put 0x5557e9ff31f0 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3208 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x5557e9ff3180 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff3110 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3d5000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff3180) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3128 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3d5000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff3110) put 0x5557e9ff3110 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3128 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x5557e9ff30a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff3030 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3d7000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff30a0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3048 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3d7000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff3030) put 0x5557e9ff3030 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff3048 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x5557e9ff2fc0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff2f50 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3d9000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2fc0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff2f68 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3d9000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff2f50) put 0x5557e9ff2f50 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff2f68 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x5557e9ff2ee0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff2e70 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3db000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2ee0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff2e88 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3db000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff2e70) put 0x5557e9ff2e70 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff2e88 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x5557e9ff2e00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff2d90 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3dd000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2e00) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff2da8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3dd000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff2d90) put 0x5557e9ff2d90 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff2da8 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x5557e9ff2d20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff2cb0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3df000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2d20) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff2cc8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3df000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff2cb0) put 0x5557e9ff2cb0 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff2cc8 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x5557e9ff2c40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff03f0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3e1000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff2c40) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff0408 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3e1000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff03f0) put 0x5557e9ff03f0 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff0408 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x5557e9ff09a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff0af0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3e3000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff09a0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff0b08 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3e3000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff0af0) put 0x5557e9ff0af0 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff0b08 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x5557e9ff0930 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7669730 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3e5000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff0930) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7669748 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3e5000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7669730) put 0x5557e7669730 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7669748 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x5557e9d921c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7a29500 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3e7000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9d921c0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7a29518 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3e7000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7a29500) put 0x5557e7a29500 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7a29518 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x5557e7452230 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e74521c0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3e9000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452230) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74521d8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3e9000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e74521c0) put 0x5557e74521c0 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74521d8 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x5557e74522a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7452310 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3eb000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74522a0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452328 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3eb000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7452310) put 0x5557e7452310 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452328 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x5557e74523f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7452150 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3ed000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74523f0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452168 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3ed000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7452150) put 0x5557e7452150 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452168 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x5557e74520e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7452070 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3ef000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74520e0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452088 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3ef000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7452070) put 0x5557e7452070 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452088 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x5557e7452460 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e74524d0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3f1000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452460) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74524e8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3f1000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e74524d0) put 0x5557e74524d0 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74524e8 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x5557e7452540 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e74525b0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3f3000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452540) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74525c8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3f3000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e74525b0) put 0x5557e74525b0 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74525c8 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x5557e7452620 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7452690 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3f5000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452620) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74526a8 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3f5000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7452690) put 0x5557e7452690 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e74526a8 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x5557e9ff3420 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9ff0d20 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3f7000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e9ff3420) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff0d38 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3f7000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e9ff0d20) put 0x5557e9ff0d20 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e9ff0d38 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x5557e7452700 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7452770 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3f9000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e7452700) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452788 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3f9000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7452770) put 0x5557e7452770 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452788 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x5557e74527e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7452850 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3fb000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74527e0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452868 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3fb000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7452850) put 0x5557e7452850 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452868 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x5557e74528c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7452930 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3fd000~2000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74528c0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452948 in 0x5557e62695e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3fd000~2000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7452930) put 0x5557e7452930 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452948 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x30000~1000: 0x0~1000 Blob(0x5557e74529a0 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x5557e7452a10 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x3ff000~1000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.blob(0x5557e74529a0) discard_unallocated 0x0~1000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452a28 in 0x5557e62695e0) _discard 0x0~1000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x3ff000~1000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.sharedblob(0x5557e7452a10) put 0x5557e7452a10 removing self from set 0x5557e75b4df0 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore.BufferSpace(0x5557e7452a28 in 0x5557e62695e0) _clear 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.2s3_head 3#1:4c9e444c:::smithi13140796-48:head#a = 0 2019-12-07T19:34:00.662+0000 7f2b794e4700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.2s3_head 3#1:40000000::::head# = 0 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea18db00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea18db00 onodes shared_blobs 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea18db00 allocated 0x[] released 0x[3cf000~31000] 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0x3cf000~31000 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 freelist release 0x3cf000~31000 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18db00 prepare 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea18db00 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18db00 io_done 2019-12-07T19:34:00.662+0000 7f2b794e4700 10 osd.6 23 dequeue_op 0x5557e9bf66e0 finish 2019-12-07T19:34:00.662+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.662+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.662+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.662+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1250 < min_flush_size 524288 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d400 had 1 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9c7d680 had 1 2019-12-07T19:34:00.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:34:00.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:34:00.663+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.663+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x85a0c~4f9 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.663+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x85a0c 0x85a0c~4f9 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x55a0c 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa0c 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xf05 and padding block with 0xfb 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x85f05 2019-12-07T19:34:00.663+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.663+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.663+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000083466s (0.000000062s flush + 0.000083404s kv commit) 2019-12-07T19:34:00.663+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557ea18db00> 2019-12-07T19:34:00.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.663+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18db00 kv_submitted 2019-12-07T19:34:00.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea18db00 2019-12-07T19:34:00.663+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18db00 finishing 2019-12-07T19:34:00.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557ea18db00 onodes 2019-12-07T19:34:00.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea18db00 done 2019-12-07T19:34:00.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678000 q now empty 2019-12-07T19:34:00.663+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557ea18db00 [3cf000~31000] 2019-12-07T19:34:00.663+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0x3cf000~31000 2019-12-07T19:34:00.663+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.663+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.663+0000 7f2b754dc700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.663+0000 7f2b794e4700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 -- 0x5557e7683200 con 0x5557e79cb800 2019-12-07T19:34:00.663+0000 7f2b794e4700 20 osd.6 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.663+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.1 v2:172.21.15.131:6823/37078 122 ==== MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=94, reqid=client.4540.0:181, at_version=0'0, trim_to=0'0, roll_forward_to=23'14)) v2 ==== 909+0+0 (crc 0 0 0) 0x5557ea034000 con 0x5557e79cac00 2019-12-07T19:34:00.663+0000 7f2b957b2700 15 osd.6 23 enqueue_op 0x5557e75c5080 prio 127 cost 0 latency 0.000024 epoch 23 MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=94, reqid=client.4540.0:181, at_version=0'0, trim_to=0'0, roll_forward_to=23'14)) v2 2019-12-07T19:34:00.663+0000 7f2b957b2700 20 osd.6 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=94, reqid=client.4540.0:181, at_version=0'0, trim_to=0'0, roll_forward_to=23'14)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.663+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=94, reqid=client.4540.0:181, at_version=0'0, trim_to=0'0, roll_forward_to=23'14)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.663+0000 7f2b764de700 20 osd.6 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7f2b764de700 20 osd.6 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=94, reqid=client.4540.0:181, at_version=0'0, trim_to=0'0, roll_forward_to=23'14)) v2) prio 127 cost 0 e23) pg 0x5557e76ba000 2019-12-07T19:34:00.663+0000 7f2b764de700 10 osd.6 23 dequeue_op 0x5557e75c5080 prio 127 cost 0 latency 0.000109 MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=94, reqid=client.4540.0:181, at_version=0'0, trim_to=0'0, roll_forward_to=23'14)) v2 pg pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.663+0000 7f2b764de700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 23/18 ECSubWrite(tid=94, reqid=client.4540.0:181, at_version=0'0, trim_to=0'0, roll_forward_to=23'14)) v2 2019-12-07T19:34:00.663+0000 7f2b764de700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=21'13 active mbc={} ps=[2~1]] append_log log((0'0,23'14], crt=21'13) [] 2019-12-07T19:34:00.663+0000 7f2b764de700 20 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=23'14 active mbc={} ps=[2~1]] rollforward: entry=23'14 (21'3) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:181 0.000000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:00.663+0000 7f2b764de700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=23'14 active mbc={} ps=[2~1]] append_log approx pg log length = 14 2019-12-07T19:34:00.663+0000 7f2b764de700 10 osd.6 pg_epoch: 23 pg[1.8s5( v 23'14 (0'0,23'14] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,2,3,6]p1(0) r=5 lpr=18 luod=0'0 crt=23'14 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.663+0000 7f2b764de700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.663+0000 7f2b764de700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.664+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e76a8960 1.8s5_head 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678a00 = 0x5557e9d5ac00 seq 28 2019-12-07T19:34:00.664+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.8s5_head 5#1:131421e8:::smithi13140796-7:head#e onode 0x5557e9bc0f00 txc 0x5557e9d5ac00 2019-12-07T19:34:00.664+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.8s5_head 5#1:131421e8:::smithi13140796-7:head#e 0x0 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57f80) put_ref 0x0~2000 Blob(0x5557e9b57f80 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621180 sbid 0x0)) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57420) put_ref 0x0~2000 Blob(0x5557e9b57420 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57ea0 sbid 0x0)) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57e30) put_ref 0x0~2000 Blob(0x5557e9b57e30 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b57340 sbid 0x0)) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.blob(0x5557e9b577a0) put_ref 0x0~1000 Blob(0x5557e9b577a0 blob([0x4d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9b57b90 sbid 0x0)) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.extentmap(0x5557e9bc1008) dirty_range mark inline shard dirty 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e9b57f80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7621180 sbid 0x0)) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x47000~2000] 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57f80) discard_unallocated 0x0~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e7621198 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x47000~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e7621180) put 0x5557e7621180 removing self from set 0x5557e76a8a30 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e7621198 in 0x5557e6269960) _clear 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e9b57420 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9b57ea0 sbid 0x0)) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x49000~2000] 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57420) discard_unallocated 0x0~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57eb8 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x49000~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9b57ea0) put 0x5557e9b57ea0 removing self from set 0x5557e76a8a30 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57eb8 in 0x5557e6269960) _clear 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x5557e9b57e30 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9b57340 sbid 0x0)) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x4b000~2000] 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.blob(0x5557e9b57e30) discard_unallocated 0x0~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57358 in 0x5557e6269960) _discard 0x0~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x4b000~2000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9b57340) put 0x5557e9b57340 removing self from set 0x5557e76a8a30 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57358 in 0x5557e6269960) _clear 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x6000~1000: 0x0~1000 Blob(0x5557e9b577a0 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x5557e9b57b90 sbid 0x0)) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x4d000~1000] 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.blob(0x5557e9b577a0) discard_unallocated 0x0~1000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57ba8 in 0x5557e6269960) _discard 0x0~1000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x4d000~1000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.sharedblob(0x5557e9b57b90) put 0x5557e9b57b90 removing self from set 0x5557e76a8a30 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore.BufferSpace(0x5557e9b57ba8 in 0x5557e6269960) _clear 2019-12-07T19:34:00.664+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.8s5_head 5#1:131421e8:::smithi13140796-7:head#e = 0 2019-12-07T19:34:00.664+0000 7f2b764de700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'._info' <- _info 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000436'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.664+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:00.664+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9d5ac00 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9d5ac00 onodes shared_blobs 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9d5ac00 allocated 0x[] released 0x[47000~7000] 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0x47000~7000 2019-12-07T19:34:00.664+0000 7f2b764de700 10 freelist release 0x47000~7000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:00.664+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 prepare 2019-12-07T19:34:00.664+0000 7f2b764de700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9d5ac00 2019-12-07T19:34:00.664+0000 7f2b764de700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 io_done 2019-12-07T19:34:00.664+0000 7f2b764de700 10 osd.6 23 dequeue_op 0x5557e75c5080 finish 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.664+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1249 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e76ca000 had 1 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0f00 had 1 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x85f05~4f8 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x85f05 0x85f05~4f8 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x55f05 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xf05 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0x3fd and padding block with 0xc03 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x863fd 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000081771s (0.000000080s flush + 0.000081691s kv commit) 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9d5ac00> 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 kv_submitted 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9d5ac00 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9d5ac00 finishing 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9d5ac00 onodes 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9d5ac00 done 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678a00 q now empty 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9d5ac00 [47000~7000] 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0x47000~7000 2019-12-07T19:34:00.664+0000 7f2b764de700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.664+0000 7f2b7a4e6700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 23/18 ECSubWriteReply(tid=94, last_complete=23'14, committed=1, applied=1)) v2 -- 0x5557e9bf3600 con 0x5557e79cac00 2019-12-07T19:34:00.664+0000 7f2b7a4e6700 20 osd.6 op_wq(0) _process empty q, waiting 2019-12-07T19:34:00.664+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 83 ==== osd_map(23..23 src has 1..23) v4 ==== 661+0+0 (crc 0 0 0) 0x5557ea027b00 con 0x5557e79ca800 2019-12-07T19:34:00.664+0000 7f2b834f8700 20 osd.6 23 OSD::ms_dispatch: osd_map(23..23 src has 1..23) v4 2019-12-07T19:34:00.664+0000 7f2b834f8700 10 osd.6 23 do_waiters -- start 2019-12-07T19:34:00.664+0000 7f2b834f8700 10 osd.6 23 do_waiters -- finish 2019-12-07T19:34:00.664+0000 7f2b834f8700 20 osd.6 23 _dispatch 0x5557ea027b00 osd_map(23..23 src has 1..23) v4 2019-12-07T19:34:00.664+0000 7f2b834f8700 3 osd.6 23 handle_osd_map epochs [23,23], i have 23, src has [1,23] 2019-12-07T19:34:00.664+0000 7f2b834f8700 10 osd.6 23 no new maps here, dropping 2019-12-07T19:34:00.664+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 84 ==== MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2 ==== 1360+0+0 (crc 0 0 0) 0x5557ea033800 con 0x5557e79ca800 2019-12-07T19:34:00.664+0000 7f2b957b2700 15 osd.6 23 enqueue_op 0x5557e9d842c0 prio 127 cost 0 latency 0.000022 epoch 23 MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:34:00.664+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e23) pg 0x5557e769c000 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 23 dequeue_op 0x5557e9d842c0 prio 127 cost 0 latency 0.000049 MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2 pg pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 23 maybe_share_map con 0x5557e79ca800 v2:172.21.15.131:6812/37076 map epoch 22 -> 23 (as per caller) 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=6 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=47, reqid=client.4540.0:184, at_version=23'13, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 21'12 (0'0,21'12] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] append_log log((0'0,21'12], crt=21'12) [23'13 (21'3) delete 1:657ac0fe:::smithi13140796-4:head by client.4540.0:184 2019-12-07T19:34:00.663938+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 0, new_object: 0] 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[2~1]] add_log_entry 23'13 (21'3) delete 1:657ac0fe:::smithi13140796-4:head by client.4540.0:184 2019-12-07T19:34:00.663938+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[2~1]] append_log approx pg log length = 13 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.664+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557ea18c600 seq 25 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) get_onode oid 1#1:657ac0fe:::smithi13140796-4:head#d key 0x818000000000000001657ac0fe21736d'ithi13140796-4!='0xfffffffffffffffe000000000000000d'o' 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.6s1_head 0x5557e75e4000) r -2 v.len 0 2019-12-07T19:34:00.664+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# -> 1#1:657ac0fe:::smithi13140796-4:head#d 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head# -> 1#1:657ac0fe:::smithi13140796-4:head#d = 0 2019-12-07T19:34:00.664+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.0000000023.00000000000000000013' <- 0000000023.00000000000000000013 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea18c600 cost 5801 (1 ios * 4000 + 1801 bytes) 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea18c600 onodes 0x5557e9bc0500 shared_blobs 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0608) update 1#1:657ac0fe:::smithi13140796-4:head#d 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:657ac0fe:::smithi13140796-4:head#d is 530 (483 bytes onode + 2 bytes spanning blobs + 45 bytes inline extents) 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea18c600 allocated 0x[] released 0x[] 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c600 prepare 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea18c600 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c600 io_done 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 lua=21'12 crt=21'12 lcod 21'12 active mbc={} ps=[2~1]] op_applied version 23'13 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2009 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f2b734d8700 10 osd.6 23 dequeue_op 0x5557e9d842c0 finish 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0500 had 1 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9d59900 had 1 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2032 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2032 < min_flush_size 524288 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x863fd~7f0 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x863fd 0x863fd~7f0 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x563fd 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0x3fd 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xbed and padding block with 0x413 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x86bed 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.664+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000104027s (0.000000059s flush + 0.000103968s kv commit) 2019-12-07T19:34:00.664+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557ea18c600> 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c600 kv_submitted 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea18c600 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c600 finishing 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557ea18c600 onodes 0x5557e9bc0500 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea18c600 done 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557ea18c600 [] 2019-12-07T19:34:00.664+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.664+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.664+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.664+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 23/18 ECSubWriteReply(tid=47, last_complete=23'13, committed=1, applied=1)) v2 -- 0x5557e9bf2200 con 0x5557e79ca800 2019-12-07T19:34:00.664+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.665+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.3 v2:172.21.15.131:6812/37076 85 ==== MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2 ==== 909+0+0 (crc 0 0 0) 0x5557e79e4800 con 0x5557e79ca800 2019-12-07T19:34:00.665+0000 7f2b957b2700 15 osd.6 23 enqueue_op 0x5557e9d84c60 prio 127 cost 0 latency 0.000017 epoch 23 MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2 2019-12-07T19:34:00.665+0000 7f2b957b2700 20 osd.6 op_wq(6) _enqueue OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 osd.6 op_wq(6) _process 1.6s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 osd.6 op_wq(6) _process OpSchedulerItem(1.6s1 PGOpItem(op=MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2) prio 127 cost 0 e23) pg 0x5557e769c000 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 osd.6 23 dequeue_op 0x5557e9d84c60 prio 127 cost 0 latency 0.000036 MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2 pg pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.6s1 23/18 ECSubWrite(tid=48, reqid=client.4540.0:184, at_version=0'0, trim_to=0'0, roll_forward_to=23'13)) v2 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] append_log log((0'0,23'13], crt=21'12) [] 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] rollforward: entry=23'13 (21'3) delete 1:657ac0fe:::smithi13140796-4:head by client.4540.0:184 2019-12-07T19:34:00.663938+0000 0 ObjectCleanRegions clean_offsets: [65536~18446744073709486079], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] append_log approx pg log length = 13 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 osd.6 pg_epoch: 23 pg[1.6s1( v 23'13 (0'0,23'13] local-lis/les=18/19 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,6,7,1,2,5]p3(0) r=1 lpr=18 luod=0'0 crt=23'13 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.665+0000 7f2b734d8700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e75e4000 1.6s1_head 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e7678640 = 0x5557ea18c900 seq 26 2019-12-07T19:34:00.665+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head#d onode 0x5557e9bc0500 txc 0x5557ea18c900 2019-12-07T19:34:00.665+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head#d 0x0 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.blob(0x5557e7a28a80) put_ref 0x0~2000 Blob(0x5557e7a28a80 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7621030 sbid 0x0)) 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.blob(0x5557e7620460) put_ref 0x0~2000 Blob(0x5557e7620460 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e7a28460 sbid 0x0)) 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.extentmap(0x5557e9bc0608) dirty_range mark inline shard dirty 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e7a28a80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7621030 sbid 0x0)) 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x43000~2000] 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.blob(0x5557e7a28a80) discard_unallocated 0x0~2000 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7621048 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 _rm buffer(0x5557e9cd74d0 space 0x5557e7621048 0x0~1000 clean) 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x43000~2000 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.sharedblob(0x5557e7621030) put 0x5557e7621030 removing self from set 0x5557e75e40d0 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7621048 in 0x5557e6268fc0) _clear 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e7620460 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e7a28460 sbid 0x0)) 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x45000~2000] 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.blob(0x5557e7620460) discard_unallocated 0x0~2000 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7a28478 in 0x5557e6268fc0) _discard 0x0~2000 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x45000~2000 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.sharedblob(0x5557e7a28460) put 0x5557e7a28460 removing self from set 0x5557e75e40d0 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore.BufferSpace(0x5557e7a28478 in 0x5557e6268fc0) _clear 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.6s1_head 1#1:657ac0fe:::smithi13140796-4:head#d = 0 2019-12-07T19:34:00.665+0000 7f2b734d8700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.6s1_head 1#1:60000000::::head# = 0 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557ea18c900 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557ea18c900 onodes shared_blobs 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557ea18c900 allocated 0x[] released 0x[43000~4000] 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0x43000~4000 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 freelist release 0x43000~4000 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c900 prepare 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557ea18c900 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c900 io_done 2019-12-07T19:34:00.665+0000 7f2b734d8700 10 osd.6 23 dequeue_op 0x5557e9d84c60 finish 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1249 < min_flush_size 524288 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795d900 had 1 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e9bc0500 had 1 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x86bed~4f8 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x86bed 0x86bed~4f8 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x56bed 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xbed 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xe5 and padding block with 0xf1b 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x870e5 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.665+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000114524s (0.000000053s flush + 0.000114471s kv commit) 2019-12-07T19:34:00.665+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.665+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.665+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557ea18c900> 2019-12-07T19:34:00.665+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.665+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c900 kv_submitted 2019-12-07T19:34:00.665+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557ea18c900 2019-12-07T19:34:00.665+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557ea18c900 finishing 2019-12-07T19:34:00.665+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557ea18c900 onodes 2019-12-07T19:34:00.665+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557ea18c900 done 2019-12-07T19:34:00.665+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e7678640 q now empty 2019-12-07T19:34:00.665+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557ea18c900 [43000~4000] 2019-12-07T19:34:00.665+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0x43000~4000 2019-12-07T19:34:00.665+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.665+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.665+0000 7f2b734d8700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.665+0000 7f2b774e0700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.6s0 23/18 ECSubWriteReply(tid=48, last_complete=23'13, committed=1, applied=1)) v2 -- 0x5557e9bf2000 con 0x5557e79ca800 2019-12-07T19:34:00.665+0000 7f2b774e0700 20 osd.6 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.668+0000 7f2b834f8700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 43 ==== osd_map(22..23 src has 1..23) v4 ==== 1258+0+0 (crc 0 0 0) 0x5557e74cb8c0 con 0x5557e7519800 2019-12-07T19:34:00.668+0000 7f2b834f8700 20 osd.6 23 OSD::ms_dispatch: osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.668+0000 7f2b834f8700 10 osd.6 23 do_waiters -- start 2019-12-07T19:34:00.668+0000 7f2b834f8700 10 osd.6 23 do_waiters -- finish 2019-12-07T19:34:00.668+0000 7f2b834f8700 20 osd.6 23 _dispatch 0x5557e74cb8c0 osd_map(22..23 src has 1..23) v4 2019-12-07T19:34:00.668+0000 7f2b834f8700 3 osd.6 23 handle_osd_map epochs [22,23], i have 23, src has [1,23] 2019-12-07T19:34:00.668+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 44 ==== MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=32, reqid=client.4540.0:185, at_version=23'7, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 1364+0+0 (crc 0 0 0) 0x5557e763f000 con 0x5557e7519800 2019-12-07T19:34:00.668+0000 7f2b834f8700 10 osd.6 23 no new maps here, dropping 2019-12-07T19:34:00.668+0000 7f2b95fb3700 15 osd.6 23 enqueue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000019 epoch 23 MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=32, reqid=client.4540.0:185, at_version=23'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:34:00.668+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=32, reqid=client.4540.0:185, at_version=23'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=32, reqid=client.4540.0:185, at_version=23'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=32, reqid=client.4540.0:185, at_version=23'7, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e23) pg 0x5557e764c000 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 23 dequeue_op 0x5557e75c5b80 prio 127 cost 0 latency 0.000044 MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=32, reqid=client.4540.0:185, at_version=23'7, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 23 maybe_share_map con 0x5557e7519800 v2:172.21.15.131:6804/37077 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=32, reqid=client.4540.0:185, at_version=23'7, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] append_log log((0'0,21'6], crt=21'6) [23'7 (21'6) delete 1:a7686634:::smithi13140796-35:head by client.4540.0:185 2019-12-07T19:34:00.666376+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 0, new_object: 0] 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[2~1]] add_log_entry 23'7 (21'6) delete 1:a7686634:::smithi13140796-35:head by client.4540.0:185 2019-12-07T19:34:00.666376+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[2~1]] append_log approx pg log length = 7 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.668+0000 7f2b77ce1700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e7664900 seq 18 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.5s1_head 0x5557e6281c20) get_onode oid 1#1:a7686634:::smithi13140796-35:head#7 key 0x818000000000000001a76866'4!smithi13140796-35!='0xfffffffffffffffe0000000000000007'o' 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6).collection(1.5s1_head 0x5557e6281c20) r -2 v.len 0 2019-12-07T19:34:00.668+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# -> 1#1:a7686634:::smithi13140796-35:head#7 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _rename 1.5s1_head 1#1:a7686634:::smithi13140796-35:head# -> 1#1:a7686634:::smithi13140796-35:head#7 = 0 2019-12-07T19:34:00.668+0000 7f2b77ce1700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.0000000023.00000000000000000007' <- 0000000023.00000000000000000007 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e7664900 cost 5804 (1 ios * 4000 + 1804 bytes) 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e7664900 onodes 0x5557ea010780 shared_blobs 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore.extentmap(0x5557ea010888) update 1#1:a7686634:::smithi13140796-35:head#7 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _record_onode onode 1#1:a7686634:::smithi13140796-35:head#7 is 833 (485 bytes onode + 2 bytes spanning blobs + 346 bytes inline extents) 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e7664900 allocated 0x[] released 0x[] 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 prepare 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e7664900 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 io_done 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active mbc={} ps=[2~1]] op_applied version 23'7 2019-12-07T19:34:00.668+0000 7f2b77ce1700 10 osd.6 23 dequeue_op 0x5557e75c5b80 finish 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2315 < min_flush_size 524288 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea010780 had 1 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea010a00 had 1 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2338 < min_flush_size 524288 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 2338 < min_flush_size 524288 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x870e5~922 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x870e5 0x870e5~922 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x570e5 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xe5 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xa07 and padding block with 0x5f9 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x87a07 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.668+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000079185s (0.000000042s flush + 0.000079143s kv commit) 2019-12-07T19:34:00.668+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e7664900> 2019-12-07T19:34:00.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.668+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 kv_submitted 2019-12-07T19:34:00.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e7664900 2019-12-07T19:34:00.668+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e7664900 finishing 2019-12-07T19:34:00.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e7664900 onodes 0x5557ea010780 2019-12-07T19:34:00.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e7664900 done 2019-12-07T19:34:00.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:34:00.668+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e7664900 [] 2019-12-07T19:34:00.668+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.668+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.668+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.668+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 23/18 ECSubWriteReply(tid=32, last_complete=23'7, committed=1, applied=1)) v2 -- 0x5557e9ef3200 con 0x5557e7519800 2019-12-07T19:34:00.668+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.669+0000 7f2b95fb3700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] <== osd.2 v2:172.21.15.131:6804/37077 45 ==== MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=33, reqid=client.4540.0:185, at_version=0'0, trim_to=0'0, roll_forward_to=23'7)) v2 ==== 910+0+0 (crc 0 0 0) 0x5557e79e4000 con 0x5557e7519800 2019-12-07T19:34:00.669+0000 7f2b95fb3700 15 osd.6 23 enqueue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000024 epoch 23 MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=33, reqid=client.4540.0:185, at_version=0'0, trim_to=0'0, roll_forward_to=23'7)) v2 2019-12-07T19:34:00.669+0000 7f2b95fb3700 20 osd.6 op_wq(5) _enqueue OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=33, reqid=client.4540.0:185, at_version=0'0, trim_to=0'0, roll_forward_to=23'7)) v2) prio 127 cost 0 e23) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=33, reqid=client.4540.0:185, at_version=0'0, trim_to=0'0, roll_forward_to=23'7)) v2) prio 127 cost 0 e23) queued 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process 1.5s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process OpSchedulerItem(1.5s1 PGOpItem(op=MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=33, reqid=client.4540.0:185, at_version=0'0, trim_to=0'0, roll_forward_to=23'7)) v2) prio 127 cost 0 e23) pg 0x5557e764c000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 osd.6 23 dequeue_op 0x5557e75c4000 prio 127 cost 0 latency 0.000064 MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=33, reqid=client.4540.0:185, at_version=0'0, trim_to=0'0, roll_forward_to=23'7)) v2 pg pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.5s1 23/18 ECSubWrite(tid=33, reqid=client.4540.0:185, at_version=0'0, trim_to=0'0, roll_forward_to=23'7)) v2 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] append_log log((0'0,23'7], crt=21'6) [] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=23'7 active mbc={} ps=[2~1]] rollforward: entry=23'7 (21'6) delete 1:a7686634:::smithi13140796-35:head by client.4540.0:185 2019-12-07T19:34:00.666376+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=23'7 active mbc={} ps=[2~1]] append_log approx pg log length = 7 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 osd.6 pg_epoch: 23 pg[1.5s1( v 23'7 (0'0,23'7] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,6,3,4,1,5]p2(0) r=1 lpr=18 luod=0'0 crt=23'7 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) queue_transactions ch 0x5557e6281c20 1.5s1_head 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_create osr 0x5557e75c8280 = 0x5557e9b10c00 seq 19 2019-12-07T19:34:00.669+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.5s1_head 1#1:a7686634:::smithi13140796-35:head#7 onode 0x5557ea010780 txc 0x5557e9b10c00 2019-12-07T19:34:00.669+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _do_truncate 1.5s1_head 1#1:a7686634:::smithi13140796-35:head#7 0x0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5abd0) put_ref 0x0~2000 Blob(0x5557e9e5abd0 blob([0x257000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9b7a930 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5ab60) put_ref 0x0~2000 Blob(0x5557e9e5ab60 blob([0x259000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5aaf0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5aa10) put_ref 0x0~2000 Blob(0x5557e9e5aa10 blob([0x25b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5a9a0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5a930) put_ref 0x0~2000 Blob(0x5557e9e5a930 blob([0x25d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5a8c0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5a850) put_ref 0x0~2000 Blob(0x5557e9e5a850 blob([0x25f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5a7e0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5a770) put_ref 0x0~2000 Blob(0x5557e9e5a770 blob([0x261000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5a700 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5a690) put_ref 0x0~2000 Blob(0x5557e9e5a690 blob([0x263000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9c2bdc0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0a930) put_ref 0x0~2000 Blob(0x5557e9f0a930 blob([0x265000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0a9a0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0aa10) put_ref 0x0~2000 Blob(0x5557e9f0aa10 blob([0x267000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0aa80 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0aaf0) put_ref 0x0~2000 Blob(0x5557e9f0aaf0 blob([0x269000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0ab60 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0abd0) put_ref 0x0~2000 Blob(0x5557e9f0abd0 blob([0x26b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0ac40 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0acb0) put_ref 0x0~2000 Blob(0x5557e9f0acb0 blob([0x26d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0ad20 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0ad90) put_ref 0x0~2000 Blob(0x5557e9f0ad90 blob([0x26f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0ae00 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0ae70) put_ref 0x0~2000 Blob(0x5557e9f0ae70 blob([0x271000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0aee0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5ad90) put_ref 0x0~2000 Blob(0x5557e9e5ad90 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9e5ad20 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0af50) put_ref 0x0~2000 Blob(0x5557e9f0af50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0afc0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0b030) put_ref 0x0~2000 Blob(0x5557e9f0b030 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x5557e9f0b0a0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0b110) put_ref 0x0~1000 Blob(0x5557e9f0b110 blob([0x279000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x5557e9f0b180 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.extentmap(0x5557ea010888) dirty_range mark inline shard dirty 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x5557e9e5abd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9b7a930 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x257000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5abd0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b7a948 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 _rm buffer(0x5557e9e44000 space 0x5557e9b7a948 0x0~1000 clean) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x257000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9b7a930) put 0x5557e9b7a930 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9b7a948 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x5557e9e5ab60 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5aaf0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x259000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5ab60) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5ab08 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x259000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9e5aaf0) put 0x5557e9e5aaf0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5ab08 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x5557e9e5aa10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5a9a0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x25b000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5aa10) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5a9b8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x25b000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9e5a9a0) put 0x5557e9e5a9a0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5a9b8 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x5557e9e5a930 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5a8c0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x25d000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5a930) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5a8d8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x25d000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9e5a8c0) put 0x5557e9e5a8c0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5a8d8 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x5557e9e5a850 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5a7e0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x25f000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5a850) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5a7f8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x25f000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9e5a7e0) put 0x5557e9e5a7e0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5a7f8 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x5557e9e5a770 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5a700 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x261000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5a770) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5a718 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x261000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9e5a700) put 0x5557e9e5a700 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5a718 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x5557e9e5a690 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9c2bdc0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x263000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5a690) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9c2bdd8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x263000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9c2bdc0) put 0x5557e9c2bdc0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9c2bdd8 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x5557e9f0a930 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0a9a0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x265000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0a930) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0a9b8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x265000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0a9a0) put 0x5557e9f0a9a0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0a9b8 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x5557e9f0aa10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0aa80 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x267000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0aa10) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0aa98 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x267000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0aa80) put 0x5557e9f0aa80 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0aa98 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x5557e9f0aaf0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0ab60 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x269000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0aaf0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0ab78 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x269000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0ab60) put 0x5557e9f0ab60 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0ab78 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x5557e9f0abd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0ac40 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x26b000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0abd0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0ac58 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x26b000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0ac40) put 0x5557e9f0ac40 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0ac58 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x5557e9f0acb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0ad20 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x26d000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0acb0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0ad38 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x26d000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0ad20) put 0x5557e9f0ad20 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0ad38 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x5557e9f0ad90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0ae00 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x26f000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0ad90) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0ae18 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x26f000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0ae00) put 0x5557e9f0ae00 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0ae18 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x5557e9f0ae70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0aee0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x271000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0ae70) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0aef8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x271000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0aee0) put 0x5557e9f0aee0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0aef8 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x5557e9e5ad90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9e5ad20 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x273000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9e5ad90) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5ad38 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x273000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9e5ad20) put 0x5557e9e5ad20 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9e5ad38 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x5557e9f0af50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0afc0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x275000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0af50) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0afd8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x275000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0afc0) put 0x5557e9f0afc0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0afd8 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x5557e9f0b030 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x5557e9f0b0a0 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x277000~2000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0b030) discard_unallocated 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0b0b8 in 0x5557e6269180) _discard 0x0~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x277000~2000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0b0a0) put 0x5557e9f0b0a0 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0b0b8 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish lex_old 0x22000~1000: 0x0~1000 Blob(0x5557e9f0b110 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x5557e9f0b180 sbid 0x0)) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish blob release [0x279000~1000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.blob(0x5557e9f0b110) discard_unallocated 0x0~1000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0b198 in 0x5557e6269180) _discard 0x0~1000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _wctx_finish release 0x279000~1000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.sharedblob(0x5557e9f0b180) put 0x5557e9f0b180 removing self from set 0x5557e6281cf0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore.BufferSpace(0x5557e9f0b198 in 0x5557e6269180) _clear 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _remove 1.5s1_head 1#1:a7686634:::smithi13140796-35:head#7 = 0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 15 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _omap_setkeys 1.5s1_head 1#1:a0000000::::head# = 0 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_calc_cost 0x5557e9b10c00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_write_nodes txc 0x5557e9b10c00 onodes shared_blobs 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv txc 0x5557e9b10c00 allocated 0x[] released 0x[257000~23000] 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finalize_kv release 0x257000~23000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 freelist release 0x257000~23000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 prepare 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish_io 0x5557e9b10c00 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 io_done 2019-12-07T19:34:00.669+0000 7f2b73cd9700 10 osd.6 23 dequeue_op 0x5557e75c4000 finish 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread wake 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1250 < min_flush_size 524288 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557e795c780 had 1 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_apply_kv onode 0x5557ea010780 had 1 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs _fsync 0x5557e76ce280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs _flush 0x5557e76ce280 0x87a07~4f9 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs _flush_range 0x5557e76ce280 pos 0x87a07 0x87a07~4f9 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:00.655268+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x57a07 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluefs _flush_range using partial tail 0xa07 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluefs _flush_range caching tail of 0xf00 and padding block with 0x100 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluefs _flush_range h 0x5557e76ce280 pos now 0x87f00 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 2019-12-07T19:34:00.669+0000 7f2b864fe700 10 bluefs wait_for_aio 0x5557e76ce280 done in 0.000000 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluefs flush_bdev 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread committed 1 cleaned 0 in 0.000113635s (0.000000064s flush + 0.000113571s kv commit) 2019-12-07T19:34:00.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread wake 2019-12-07T19:34:00.669+0000 7f2b864fe700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_sync_thread sleep 2019-12-07T19:34:00.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread kv_committed <0x5557e9b10c00> 2019-12-07T19:34:00.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 kv_submitted 2019-12-07T19:34:00.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_committed_kv txc 0x5557e9b10c00 2019-12-07T19:34:00.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_state_proc txc 0x5557e9b10c00 finishing 2019-12-07T19:34:00.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish 0x5557e9b10c00 onodes 2019-12-07T19:34:00.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish txc 0x5557e9b10c00 done 2019-12-07T19:34:00.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _txc_finish osr 0x5557e75c8280 q now empty 2019-12-07T19:34:00.669+0000 7f2b8b508700 10 bluestore(/var/lib/ceph/osd/ceph-6) _txc_release_alloc(sync) 0x5557e9b10c00 [257000~23000] 2019-12-07T19:34:00.669+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release 0x257000~23000 2019-12-07T19:34:00.669+0000 7f2b8b508700 10 fbmap_alloc 0x5557e6282e00 release done 2019-12-07T19:34:00.669+0000 7f2b8b508700 20 bluestore(/var/lib/ceph/osd/ceph-6) _kv_finalize_thread sleep 2019-12-07T19:34:00.669+0000 7f2b77ce1700 1 -- [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 23/18 ECSubWriteReply(tid=33, last_complete=23'7, committed=1, applied=1)) v2 -- 0x5557e9ef3000 con 0x5557e7519800 2019-12-07T19:34:00.669+0000 7f2b77ce1700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.669+0000 7f2b73cd9700 20 osd.6 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.681+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/890522420 conn(0x5557e7450400 msgr2=0x5557e7a99180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 99 2019-12-07T19:34:00.681+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/890522420 conn(0x5557e7450400 msgr2=0x5557e7a99180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:00.681+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/890522420 conn(0x5557e7450400 0x5557e7a99180 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:00.681+0000 7f2b957b2700 1 --2- [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] >> 172.21.15.131:0/890522420 conn(0x5557e7450400 0x5557e7a99180 crc :-1 s=READY pgs=11 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:00.681+0000 7f2b88017700 2 osd.6 23 ms_handle_reset con 0x5557e7450400 session 0x5557e9c2e4e0 2019-12-07T19:34:00.698+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.748+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.798+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.848+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.861+0000 7f2b937ae700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 6 (PID: 36932) UID: 0 2019-12-07T19:34:00.899+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.948+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.962+0000 7f2b937ae700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 6 (PID: 36932) UID: 0 2019-12-07T19:34:00.999+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.049+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.071+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] <== osd.7 172.21.15.109:0/36939 7 ==== osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.690666404s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e70e2000 con 0x5557e748f800 2019-12-07T19:34:01.071+0000 7f2b957b2700 20 osd.6 23 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690319434s,18446744073.690666404s]) 2019-12-07T19:34:01.071+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.527675697s delta_ub 0.019232182s) v5 -- 0x5557e765e600 con 0x5557e748f800 2019-12-07T19:34:01.071+0000 7f2b957b2700 10 osd.6 23 maybe_share_map con 0x5557e79f4000 v2:172.21.15.109:6826/36939 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:01.072+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] <== osd.7 172.21.15.109:0/36939 7 ==== osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.690666404s) v5 ==== 2033+0+0 (crc 0 0 0) 0x5557e70e2000 con 0x5557e74cc800 2019-12-07T19:34:01.072+0000 7f2b957b2700 20 osd.6 23 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690272918s,18446744073.690666404s]) 2019-12-07T19:34:01.072+0000 7f2b957b2700 1 -- [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.527722213s delta_ub 0.019278698s) v5 -- 0x5557e9bf2200 con 0x5557e74cc800 2019-12-07T19:34:01.099+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.149+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.199+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.249+0000 7f2b8ad07700 20 bluestore.MempoolThread(0x5557e6312a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 387473 data_alloc: 671088640 data_used: 0